diff --git a/.github/workflows/cherry-pick-to-release-branch.yml b/.github/workflows/cherry-pick-to-release-branch.yml
index ed6946e8c2f11..e98e1769630b9 100644
--- a/.github/workflows/cherry-pick-to-release-branch.yml
+++ b/.github/workflows/cherry-pick-to-release-branch.yml
@@ -3,7 +3,7 @@ on:
   pull_request:
     branches:
       - main
-    types: ["closed"]
+    types: ["closed", "labeled"]
 
 jobs:
   release_pull_request_1_1:
@@ -22,3 +22,19 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  release_pull_request_1_2:
+    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-v1.2')  &&  github.event.pull_request.merged == true"
+    runs-on: ubuntu-latest
+    name: release_pull_request
+    steps:
+      - name: checkout
+        uses: actions/checkout@v1
+      - name: Create PR to branch
+        uses: risingwavelabs/github-action-cherry-pick@master
+        with:
+          pr_branch: 'v1.2-rc'
+          pr_labels: 'cherry-pick'
+          pr_body: ${{ format('Cherry picking \#{0} onto branch v1.2-rc', github.event.number) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
index 9f3b682423fce..66f740666e2a7 100644
--- a/.github/workflows/doc.yml
+++ b/.github/workflows/doc.yml
@@ -5,6 +5,11 @@ on:
     branches:
       - main
 
+env:
+  SCCACHE_GHA_ENABLED: "true"
+  RUSTC_WRAPPER: "sccache"
+  RUSTDOCFLAGS: "--cfg docsrs --markdown-css rust.css --markdown-no-toc --index-page /home/runner/work/risingwave/risingwave/docs/rustdoc/index.md -Zunstable-options"
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -18,24 +23,32 @@ jobs:
         uses: mozilla-actions/sccache-action@v0.0.3
         with:
           version: "v0.5.2"
-      - name: Docs
+      - name: build rustdocs
         run: |
-          RUSTDOCFLAGS="--markdown-css rust.css --markdown-no-toc --index-page docs/rustdoc/index.md -Zunstable-options" cargo doc --workspace --no-deps --document-private-items
+          cargo doc --workspace --no-deps --document-private-items
           cp docs/rustdoc/rust.css target/doc/rust.css
-        env:
-          SCCACHE_GHA_ENABLED: "true"
-          RUSTC_WRAPPER: "sccache"
-      - name: Fix file permissions
-        shell: sh
+
+          mkdir artifact
+          cp -R target/doc/* artifact
+      - name: Install cargo-docset
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-docset
+      - name: build docsets
         run: |
-          chmod -c -R +rX "target/doc" |
-          while read line; do
-              echo "::warning title=Invalid file permissions automatically fixed::$line"
-          done
+          cargo docset --no-clean --docset-name RisingWave
+          tar -czf risingwave.docset.tgz target/docset/risingwave.docset
+
+          echo "<entry>
+          <version>$(date -u +%Y-%m-%dT%H:%M:%SZ)</version>
+          <url>https://risingwavelabs.github.io/risingwave/risingwave.docset.tgz</url>
+          </entry>" > RisingWave.xml
+
+          cp -t artifact "risingwave.docset.tgz" "RisingWave.xml"
       - name: Upload artifacts
         uses: actions/upload-pages-artifact@v1
         with:
-          path: target/doc
+          path: artifact
 
   deploy:
     needs: build
diff --git a/CODEOWNERS b/CODEOWNERS
new file mode 100644
index 0000000000000..ee9d22f99f782
--- /dev/null
+++ b/CODEOWNERS
@@ -0,0 +1 @@
+Cargo.lock @risingwavelabs/cargo-lock
diff --git a/Cargo.lock b/Cargo.lock
index 92bb4755cdbf0..cdfa79410491f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -49,9 +49,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
+checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
 dependencies = [
  "memchr",
 ]
@@ -98,18 +98,26 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
 
+[[package]]
+name = "ansi_term"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "anstream"
-version = "0.3.2"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"
 dependencies = [
  "anstyle",
  "anstyle-parse",
  "anstyle-query",
  "anstyle-wincon",
  "colorchoice",
- "is-terminal",
  "utf8parse",
 ]
 
@@ -139,9 +147,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "1.0.2"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
+checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"
 dependencies = [
  "anstyle",
  "windows-sys 0.48.0",
@@ -149,9 +157,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.75"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
 dependencies = [
  "backtrace",
 ]
@@ -164,7 +172,7 @@ checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7"
 dependencies = [
  "apache-avro-derive",
  "byteorder",
- "digest",
+ "digest 0.10.7",
  "lazy_static",
  "libflate",
  "log",
@@ -190,7 +198,7 @@ dependencies = [
  "byteorder",
  "bzip2",
  "crc32fast",
- "digest",
+ "digest 0.10.7",
  "lazy_static",
  "libflate",
  "log",
@@ -221,7 +229,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_json",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -236,33 +244,11 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
 
-[[package]]
-name = "arrow"
-version = "45.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7104b9e9761613ae92fe770c741d6bbf1dbc791a0fe204400aebdd429875741"
-dependencies = [
- "ahash 0.8.3",
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-csv",
- "arrow-data",
- "arrow-ipc",
- "arrow-json",
- "arrow-ord",
- "arrow-row",
- "arrow-schema",
- "arrow-select",
- "arrow-string",
-]
-
 [[package]]
 name = "arrow-arith"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38e597a8e8efb8ff52c50eaf8f4d85124ce3c1bf20fab82f476d73739d9ab1c2"
+checksum = "895263144bd4a69751cbe6a34a53f26626e19770b313a9fa792c415cd0e78f11"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -275,9 +261,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a86d9c1473db72896bd2345ebb6b8ad75b8553ba390875c76708e8dc5c5492d"
+checksum = "226fdc6c3a4ae154a74c24091d36a90b514f0ed7112f5b8322c1d8f354d8e20d"
 dependencies = [
  "ahash 0.8.3",
  "arrow-buffer",
@@ -291,19 +277,20 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234b3b1c8ed00c874bf95972030ac4def6f58e02ea5a7884314388307fb3669b"
+checksum = "fc4843af4dd679c2f35b69c572874da8fde33be53eb549a5fb128e7a4b763510"
 dependencies = [
+ "bytes",
  "half 2.3.1",
  "num",
 ]
 
 [[package]]
 name = "arrow-cast"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22f61168b853c7faea8cea23a2169fdff9c82fb10ae5e2c07ad1cab8f6884931"
+checksum = "35e8b9990733a9b635f656efda3c9b8308c7a19695c9ec2c7046dd154f9b144b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -316,30 +303,11 @@ dependencies = [
  "num",
 ]
 
-[[package]]
-name = "arrow-csv"
-version = "45.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10b545c114d9bf8569c84d2fbe2020ac4eea8db462c0a37d0b65f41a90d066fe"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "chrono",
- "csv",
- "csv-core",
- "lazy_static",
- "lexical-core",
- "regex",
-]
-
 [[package]]
 name = "arrow-data"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6b6852635e7c43e5b242841c7470606ff0ee70eef323004cacc3ecedd33dd8f"
+checksum = "da900f31ff01a0a84da0572209be72b2b6f980f3ea58803635de47913191c188"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -349,16 +317,16 @@ dependencies = [
 
 [[package]]
 name = "arrow-flight"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a0b5fad0d74d4292b46579e8238c7ba93520433e026d9fec6b7873d322bb3f3"
+checksum = "e40372d37a860a742f248d4d7c137950cd793f1b46f2b99a5116c55efbe2699f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
  "arrow-ipc",
  "arrow-schema",
- "base64 0.21.2",
+ "base64 0.21.3",
  "bytes",
  "futures",
  "paste",
@@ -369,9 +337,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a66da9e16aecd9250af0ae9717ae8dd7ea0d8ca5a3e788fe3de9f4ee508da751"
+checksum = "2707a8d7ee2d345d045283ece3ae43416175873483e5d96319c929da542a0b1f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -381,46 +349,11 @@ dependencies = [
  "flatbuffers",
 ]
 
-[[package]]
-name = "arrow-json"
-version = "45.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60ee0f9d8997f4be44a60ee5807443e396e025c23cf14d2b74ce56135cb04474"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "chrono",
- "half 2.3.1",
- "indexmap 2.0.0",
- "lexical-core",
- "num",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "arrow-ord"
-version = "45.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcab05410e6b241442abdab6e1035177dc082bdb6f17049a4db49faed986d63"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
- "half 2.3.1",
- "num",
-]
-
 [[package]]
 name = "arrow-row"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91a847dd9eb0bacd7836ac63b3475c68b2210c2c96d0ec1b808237b973bd5d73"
+checksum = "0e32afc1329f7b372463b21c6ca502b07cf237e1ed420d87706c1770bb0ebd38"
 dependencies = [
  "ahash 0.8.3",
  "arrow-array",
@@ -433,37 +366,21 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54df8c47918eb634c20e29286e69494fdc20cafa5173eb6dad49c7f6acece733"
+checksum = "b104f5daa730f00fde22adc03a12aa5a2ae9ccbbf99cbd53d284119ddc90e03d"
 
 [[package]]
 name = "arrow-select"
-version = "45.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "941dbe481da043c4bd40c805a19ec2fc008846080c4953171b62bcad5ee5f7fb"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "num",
-]
-
-[[package]]
-name = "arrow-string"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "359b2cd9e071d5a3bcf44679f9d85830afebc5b9c98a08019a570a65ae933e0f"
+checksum = "73b3ca55356d1eae07cf48808d8c462cea674393ae6ad1e0b120f40b422eb2b4"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
- "arrow-select",
  "num",
- "regex",
- "regex-syntax 0.7.4",
 ]
 
 [[package]]
@@ -525,15 +442,49 @@ dependencies = [
  "event-listener",
 ]
 
+[[package]]
+name = "async-nats"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8257238e2a3629ee5618502a75d1b91f8017c24638c75349fc8d2d80cf1f7c4c"
+dependencies = [
+ "base64 0.21.3",
+ "bytes",
+ "futures",
+ "http",
+ "itoa",
+ "memchr",
+ "nkeys",
+ "nuid",
+ "once_cell",
+ "rand",
+ "regex",
+ "ring",
+ "rustls-native-certs",
+ "rustls-pemfile",
+ "rustls-webpki 0.101.4",
+ "serde",
+ "serde_json",
+ "serde_nanos",
+ "serde_repr",
+ "thiserror",
+ "time 0.3.28",
+ "tokio",
+ "tokio-retry",
+ "tokio-rustls 0.24.1",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "async-recursion"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
+checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -555,7 +506,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -572,7 +523,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -581,6 +532,17 @@ version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"
 
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "auto_enums"
 version = "0.8.2"
@@ -590,7 +552,7 @@ dependencies = [
  "derive_utils",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -657,7 +619,7 @@ dependencies = [
  "fastrand 1.9.0",
  "http",
  "hyper",
- "time 0.3.27",
+ "time 0.3.28",
  "tokio",
  "tower",
  "tracing",
@@ -853,8 +815,8 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "regex",
- "sha2",
- "time 0.3.27",
+ "sha2 0.10.7",
+ "time 0.3.28",
  "tracing",
 ]
 
@@ -887,7 +849,7 @@ dependencies = [
  "md-5",
  "pin-project-lite",
  "sha1",
- "sha2",
+ "sha2 0.10.7",
  "tracing",
 ]
 
@@ -995,7 +957,7 @@ dependencies = [
  "itoa",
  "num-integer",
  "ryu",
- "time 0.3.27",
+ "time 0.3.28",
 ]
 
 [[package]]
@@ -1120,9 +1082,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
-version = "0.21.2"
+version = "0.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"
 
 [[package]]
 name = "base64-simd"
@@ -1160,6 +1122,17 @@ version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f40afb3abbf90895dda3ddbc6d8734d24215130a22d646067690f5e318f81bc"
 
+[[package]]
+name = "bigdecimal"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "bincode"
 version = "1.3.3"
@@ -1188,6 +1161,25 @@ dependencies = [
  "virtue",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.59.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
+dependencies = [
+ "bitflags 1.3.2",
+ "cexpr",
+ "clang-sys",
+ "lazy_static",
+ "lazycell",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+]
+
 [[package]]
 name = "bit-vec"
 version = "0.6.3"
@@ -1234,6 +1226,15 @@ dependencies = [
  "triple_accel",
 ]
 
+[[package]]
+name = "block-buffer"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -1466,17 +1467,37 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
 
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cfg-or-panic"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf85d5384815558275789d91d1895d1d9919a6e2534d6144650f036ac65691a6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.31",
+]
+
 [[package]]
 name = "chrono"
-version = "0.4.26"
+version = "0.4.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
+checksum = "d87d9d13be47a5b7c3907137f1290b0459a7f80efb26be8c52afb11963bccb02"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -1485,7 +1506,7 @@ dependencies = [
  "serde",
  "time 0.1.45",
  "wasm-bindgen",
- "winapi",
+ "windows-targets 0.48.1",
 ]
 
 [[package]]
@@ -1539,40 +1560,64 @@ dependencies = [
  "half 1.8.2",
 ]
 
+[[package]]
+name = "clang-sys"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "clap"
+version = "2.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags 1.3.2",
+ "strsim 0.7.0",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
 [[package]]
 name = "clap"
-version = "4.3.22"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b417ae4361bca3f5de378294fc7472d3c4ed86a5ef9f49e93ae722f432aae8d2"
+checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6"
 dependencies = [
  "clap_builder",
  "clap_derive",
- "once_cell",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.3.22"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c90dc0f0e42c64bff177ca9d7be6fcc9ddb0f26a6e062174a61c84dd6c644d4"
+checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08"
 dependencies = [
  "anstream",
  "anstyle",
  "clap_lex",
- "once_cell",
- "strsim",
+ "strsim 0.10.0",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.3.12"
+version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050"
+checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873"
 dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1598,7 +1643,7 @@ dependencies = [
  "serde",
  "static_assertions",
  "thiserror",
- "time 0.3.27",
+ "time 0.3.28",
  "tokio",
  "url",
 ]
@@ -1736,7 +1781,7 @@ dependencies = [
  "crossbeam-utils",
  "futures",
  "hdrhistogram",
- "humantime",
+ "humantime 2.1.0",
  "prost-types",
  "serde",
  "serde_json",
@@ -1749,6 +1794,12 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "const-oid"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b"
+
 [[package]]
 name = "const-oid"
 version = "0.9.5"
@@ -1860,7 +1911,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -1872,7 +1923,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap",
+ "clap 4.4.2",
  "criterion-plot",
  "futures",
  "is-terminal",
@@ -2053,7 +2104,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b"
 dependencies = [
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
+]
+
+[[package]]
+name = "curve25519-dalek"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61"
+dependencies = [
+ "byteorder",
+ "digest 0.9.0",
+ "rand_core 0.5.1",
+ "subtle",
+ "zeroize",
 ]
 
 [[package]]
@@ -2080,7 +2144,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "scratch",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2097,7 +2161,7 @@ checksum = "59d9ffb4193dd22180b8d5747b1e095c3d9c9c665ce39b0483a488948f437e06"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2140,7 +2204,7 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
+ "strsim 0.10.0",
  "syn 1.0.109",
 ]
 
@@ -2154,7 +2218,7 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
+ "strsim 0.10.0",
  "syn 1.0.109",
 ]
 
@@ -2168,8 +2232,8 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
- "syn 2.0.29",
+ "strsim 0.10.0",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2202,7 +2266,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2222,9 +2286,9 @@ source = "git+https://github.com/risingwavelabs/darwin-libproc.git?rev=a502be24b
 
 [[package]]
 name = "dashmap"
-version = "5.5.1"
+version = "5.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if",
  "hashbrown 0.14.0",
@@ -2233,6 +2297,12 @@ dependencies = [
  "parking_lot_core 0.9.8",
 ]
 
+[[package]]
+name = "data-encoding"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
+
 [[package]]
 name = "data-url"
 version = "0.2.0"
@@ -2248,22 +2318,31 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "der"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4"
+dependencies = [
+ "const-oid 0.6.2",
+]
+
 [[package]]
 name = "der"
 version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c"
 dependencies = [
- "const-oid",
- "pem-rfc7468",
+ "const-oid 0.9.5",
+ "pem-rfc7468 0.7.0",
  "zeroize",
 ]
 
 [[package]]
 name = "deranged"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929"
+checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
 dependencies = [
  "serde",
 ]
@@ -2307,7 +2386,7 @@ checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2334,14 +2413,23 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
- "block-buffer",
- "const-oid",
+ "block-buffer 0.10.4",
+ "const-oid 0.9.5",
  "crypto-common",
  "subtle",
 ]
@@ -2396,7 +2484,7 @@ dependencies = [
  "rust_decimal",
  "serde",
  "thiserror",
- "time 0.3.27",
+ "time 0.3.28",
 ]
 
 [[package]]
@@ -2411,6 +2499,27 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49457524c7e65648794c98283282a0b7c73b10018e7091f1cdcfff314fd7ae59"
 
+[[package]]
+name = "ed25519"
+version = "1.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7"
+dependencies = [
+ "signature 1.6.4",
+]
+
+[[package]]
+name = "ed25519-dalek"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d"
+dependencies = [
+ "curve25519-dalek",
+ "ed25519",
+ "sha2 0.9.9",
+ "zeroize",
+]
+
 [[package]]
 name = "educe"
 version = "0.4.22"
@@ -2453,7 +2562,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2473,7 +2582,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -2486,7 +2595,20 @@ dependencies = [
  "num-traits",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
+dependencies = [
+ "atty",
+ "humantime 1.3.0",
+ "log",
+ "regex",
+ "termcolor",
 ]
 
 [[package]]
@@ -2542,9 +2664,9 @@ dependencies = [
 
 [[package]]
 name = "ethnum"
-version = "1.3.2"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0198b9d0078e0f30dedc7acbb21c974e838fc8fae3ee170128658a98cb2c1c04"
+checksum = "6c8ff382b2fa527fb7fb06eeebfc5bbb3f17e3cc6b9d70b006c41daa8824adac"
 dependencies = [
  "serde",
 ]
@@ -2651,11 +2773,12 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.26"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
+checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
 dependencies = [
  "crc32fast",
+ "libz-sys",
  "miniz_oxide",
 ]
 
@@ -2710,7 +2833,7 @@ dependencies = [
 [[package]]
 name = "foyer"
 version = "0.1.0"
-source = "git+https://github.com/xxchan/foyer?rev=fc5e4cb#fc5e4cb3dadddc8fbab531c3e5aec7a0b4e6f670"
+source = "git+https://github.com/mrcroxx/foyer?rev=2b8907c#2b8907c3cc629da22622a17bbc6bf16dc52c7bc1"
 dependencies = [
  "async-trait",
  "bytes",
@@ -2724,7 +2847,7 @@ dependencies = [
  "itertools 0.11.0",
  "libc",
  "memoffset 0.9.0",
- "nix",
+ "nix 0.27.1",
  "parking_lot 0.12.1",
  "paste",
  "prometheus",
@@ -2738,7 +2861,7 @@ dependencies = [
 [[package]]
 name = "foyer-common"
 version = "0.1.0"
-source = "git+https://github.com/xxchan/foyer?rev=fc5e4cb#fc5e4cb3dadddc8fbab531c3e5aec7a0b4e6f670"
+source = "git+https://github.com/mrcroxx/foyer?rev=2b8907c#2b8907c3cc629da22622a17bbc6bf16dc52c7bc1"
 dependencies = [
  "bytes",
  "foyer-workspace-hack",
@@ -2752,7 +2875,7 @@ dependencies = [
 [[package]]
 name = "foyer-intrusive"
 version = "0.1.0"
-source = "git+https://github.com/xxchan/foyer?rev=fc5e4cb#fc5e4cb3dadddc8fbab531c3e5aec7a0b4e6f670"
+source = "git+https://github.com/mrcroxx/foyer?rev=2b8907c#2b8907c3cc629da22622a17bbc6bf16dc52c7bc1"
 dependencies = [
  "bytes",
  "cmsketch",
@@ -2770,7 +2893,7 @@ dependencies = [
 [[package]]
 name = "foyer-storage"
 version = "0.1.0"
-source = "git+https://github.com/xxchan/foyer?rev=fc5e4cb#fc5e4cb3dadddc8fbab531c3e5aec7a0b4e6f670"
+source = "git+https://github.com/mrcroxx/foyer?rev=2b8907c#2b8907c3cc629da22622a17bbc6bf16dc52c7bc1"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -2786,7 +2909,7 @@ dependencies = [
  "itertools 0.11.0",
  "libc",
  "memoffset 0.9.0",
- "nix",
+ "nix 0.27.1",
  "parking_lot 0.12.1",
  "paste",
  "pin-project",
@@ -2801,24 +2924,28 @@ dependencies = [
 [[package]]
 name = "foyer-workspace-hack"
 version = "0.1.0"
-source = "git+https://github.com/xxchan/foyer?rev=fc5e4cb#fc5e4cb3dadddc8fbab531c3e5aec7a0b4e6f670"
+source = "git+https://github.com/mrcroxx/foyer?rev=2b8907c#2b8907c3cc629da22622a17bbc6bf16dc52c7bc1"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-utils",
  "either",
  "futures-channel",
  "futures-core",
+ "futures-executor",
  "futures-sink",
+ "futures-util",
  "itertools 0.10.5",
- "libc",
- "memchr",
- "miniz_oxide",
+ "lock_api",
+ "nix 0.27.1",
  "parking_lot 0.12.1",
  "parking_lot_core 0.9.8",
  "proc-macro2",
  "quote",
  "rand",
- "syn 2.0.29",
+ "regex",
+ "regex-automata 0.3.8",
+ "regex-syntax 0.7.5",
+ "syn 2.0.31",
  "tokio",
  "tracing-core",
 ]
@@ -2829,6 +2956,58 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
 
+[[package]]
+name = "frunk"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11a351b59e12f97b4176ee78497dff72e4276fb1ceb13e19056aca7fa0206287"
+dependencies = [
+ "frunk_core",
+ "frunk_derives",
+ "frunk_proc_macros",
+]
+
+[[package]]
+name = "frunk_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af2469fab0bd07e64ccf0ad57a1438f63160c69b2e57f04a439653d68eb558d6"
+
+[[package]]
+name = "frunk_derives"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e"
+dependencies = [
+ "frunk_proc_macro_helpers",
+ "quote",
+ "syn 2.0.31",
+]
+
+[[package]]
+name = "frunk_proc_macro_helpers"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35b54add839292b743aeda6ebedbd8b11e93404f902c56223e51b9ec18a13d2c"
+dependencies = [
+ "frunk_core",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.31",
+]
+
+[[package]]
+name = "frunk_proc_macros"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71b85a1d4a9a6b300b41c05e8e13ef2feca03e0334127f29eca9506a7fe13a93"
+dependencies = [
+ "frunk_core",
+ "frunk_proc_macro_helpers",
+ "quote",
+ "syn 2.0.31",
+]
+
 [[package]]
 name = "fs-err"
 version = "2.9.0"
@@ -2959,7 +3138,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -3061,7 +3240,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "931bedb2264cb00f914b0a6a5c304e34865c34306632d3932e0951a073e4a67d"
 dependencies = [
  "async-trait",
- "base64 0.21.2",
+ "base64 0.21.3",
  "google-cloud-metadata",
  "google-cloud-token",
  "home",
@@ -3070,7 +3249,7 @@ dependencies = [
  "serde",
  "serde_json",
  "thiserror",
- "time 0.3.27",
+ "time 0.3.28",
  "tokio",
  "tracing",
  "urlencoding",
@@ -3116,9 +3295,9 @@ dependencies = [
 
 [[package]]
 name = "google-cloud-pubsub"
-version = "0.19.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f99771850b248cb3e2ac957eb7d321155d896792a141cee1519b4c4aa8e9e74d"
+checksum = "fc1f6c87f794b93fbb253f4983c19cd8a810b7fa067a1401ce4cf91ede758b34"
 dependencies = [
  "async-channel",
  "async-stream",
@@ -3142,6 +3321,23 @@ dependencies = [
  "async-trait",
 ]
 
+[[package]]
+name = "governor"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "821239e5672ff23e2a7060901fa622950bbd80b649cdaadd78d1c1767ed14eb4"
+dependencies = [
+ "cfg-if",
+ "dashmap",
+ "futures",
+ "futures-timer",
+ "no-std-compat",
+ "nonzero_ext",
+ "parking_lot 0.12.1",
+ "rand",
+ "smallvec",
+]
+
 [[package]]
 name = "h2"
 version = "0.3.20"
@@ -3245,6 +3441,15 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.2"
@@ -3263,7 +3468,7 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
 dependencies = [
- "digest",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -3326,6 +3531,15 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
+dependencies = [
+ "quick-error",
+]
+
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -3449,11 +3663,16 @@ dependencies = [
 [[package]]
 name = "icelake"
 version = "0.0.9"
-source = "git+https://github.com/icelake-io/icelake?rev=ec7d251e5e6db0673cea35edd62e6c7df81c4759#ec7d251e5e6db0673cea35edd62e6c7df81c4759"
+source = "git+https://github.com/icelake-io/icelake?rev=a6790d17094754959e351fac1e11147e37643e97#a6790d17094754959e351fac1e11147e37643e97"
 dependencies = [
  "anyhow",
  "apache-avro 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "arrow",
+ "arrow-arith",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-row",
+ "arrow-schema",
+ "arrow-select",
  "async-trait",
  "bitvec",
  "bytes",
@@ -3468,6 +3687,7 @@ dependencies = [
  "regex",
  "rust_decimal",
  "serde",
+ "serde_bytes",
  "serde_json",
  "serde_with 3.3.0",
  "tokio",
@@ -3587,7 +3807,7 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.2",
  "libc",
  "windows-sys 0.48.0",
 ]
@@ -3604,7 +3824,7 @@ version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.2",
  "rustix 0.38.8",
  "windows-sys 0.48.0",
 ]
@@ -3679,13 +3899,29 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonschema-transpiler"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40dc049cde84638e2a5657f45a4c74f471a4ecf2a8e2aa0b8ea899d5b1ebdee9"
+dependencies = [
+ "clap 2.32.0",
+ "env_logger",
+ "heck 0.3.3",
+ "log",
+ "maplit",
+ "regex",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "jsonwebtoken"
 version = "8.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.3",
  "pem",
  "ring",
  "serde",
@@ -3711,6 +3947,21 @@ dependencies = [
  "spin 0.5.2",
 ]
 
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
+[[package]]
+name = "lexical"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6"
+dependencies = [
+ "lexical-core",
+]
+
 [[package]]
 name = "lexical-core"
 version = "0.8.5"
@@ -3801,6 +4052,16 @@ dependencies = [
  "rle-decode-fast",
 ]
 
+[[package]]
+name = "libloading"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.7"
@@ -3813,7 +4074,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7"
 dependencies = [
- "clap",
+ "clap 4.4.2",
  "termcolor",
  "threadpool",
 ]
@@ -3910,6 +4171,15 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
+[[package]]
+name = "lru"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909"
+dependencies = [
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "lru"
 version = "0.10.1"
@@ -4041,7 +4311,7 @@ dependencies = [
 [[package]]
 name = "madsim-rdkafka"
 version = "0.2.22"
-source = "git+https://github.com/madsim-rs/madsim.git?rev=f01e3ca#f01e3cacfeebfec9893a3f94a57386d1c7ba4dc5"
+source = "git+https://github.com/madsim-rs/madsim.git?rev=bb8f063#bb8f06384517ea3950b6c7a29a32c233058b89c7"
 dependencies = [
  "async-channel",
  "async-trait",
@@ -4139,7 +4409,7 @@ version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"
 dependencies = [
- "digest",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -4150,9 +4420,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
 
 [[package]]
 name = "memcomparable"
@@ -4298,9 +4568,80 @@ checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e"
 name = "multimap"
 version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "mysql_async"
+version = "0.31.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2975442c70450b8f3a0400216321f6ab7b8bda177579f533d312ac511f913655"
+dependencies = [
+ "bytes",
+ "crossbeam",
+ "flate2",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "lazy_static",
+ "lru 0.8.1",
+ "mio",
+ "mysql_common",
+ "native-tls",
+ "once_cell",
+ "pem",
+ "percent-encoding",
+ "pin-project",
+ "priority-queue",
+ "serde",
+ "serde_json",
+ "socket2 0.4.9",
+ "thiserror",
+ "tokio",
+ "tokio-native-tls",
+ "tokio-util",
+ "twox-hash",
+ "url",
+]
+
+[[package]]
+name = "mysql_common"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9006c95034ccf7b903d955f210469119f6c3477fc9c9e7a7845ce38a3e665c2a"
 dependencies = [
+ "base64 0.13.1",
+ "bigdecimal",
+ "bindgen",
+ "bitflags 1.3.2",
+ "bitvec",
+ "byteorder",
+ "bytes",
+ "cc",
+ "chrono",
+ "cmake",
+ "crc32fast",
+ "flate2",
+ "frunk",
+ "lazy_static",
+ "lexical",
+ "num-bigint",
+ "num-traits",
+ "rand",
+ "regex",
+ "rust_decimal",
+ "saturating",
  "serde",
+ "serde_json",
+ "sha1",
+ "sha2 0.10.7",
+ "smallvec",
+ "subprocess",
+ "thiserror",
+ "time 0.3.28",
+ "uuid",
 ]
 
 [[package]]
@@ -4361,6 +4702,39 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "nix"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
+dependencies = [
+ "bitflags 2.4.0",
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "nkeys"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e9261eb915c785ea65708bc45ef43507ea46914e1a73f1412d1a38aba967c8e"
+dependencies = [
+ "byteorder",
+ "data-encoding",
+ "ed25519",
+ "ed25519-dalek",
+ "getrandom",
+ "log",
+ "rand",
+ "signatory",
+]
+
+[[package]]
+name = "no-std-compat"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -4371,6 +4745,12 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "nonzero_ext"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
+
 [[package]]
 name = "normalize-line-endings"
 version = "0.3.0"
@@ -4396,6 +4776,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "nuid"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20c1bb65186718d348306bf1afdeb20d9ab45b2ab80fb793c0fdcf59ffbb4f38"
+dependencies = [
+ "lazy_static",
+ "rand",
+]
+
 [[package]]
 name = "num"
 version = "0.4.1"
@@ -4412,9 +4802,9 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
 dependencies = [
  "autocfg",
  "num-integer",
@@ -4506,7 +4896,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.2",
  "libc",
 ]
 
@@ -4561,7 +4951,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_path_to_error",
- "sha2",
+ "sha2 0.10.7",
  "thiserror",
  "url",
 ]
@@ -4587,6 +4977,12 @@ version = "11.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
 [[package]]
 name = "opendal"
 version = "0.39.0"
@@ -4597,7 +4993,7 @@ dependencies = [
  "async-compat",
  "async-trait",
  "backon",
- "base64 0.21.2",
+ "base64 0.21.3",
  "bytes",
  "chrono",
  "flagset",
@@ -4615,7 +5011,7 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
- "sha2",
+ "sha2 0.10.7",
  "tokio",
  "uuid",
 ]
@@ -4649,11 +5045,11 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.56"
+version = "0.10.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
+checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.4.0",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -4670,7 +5066,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -4681,18 +5077,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "111.27.0+1.1.1v"
+version = "300.1.3+3.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02"
+checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.91"
+version = "0.9.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
+checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
 dependencies = [
  "cc",
  "libc",
@@ -4911,9 +5307,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "45.0.0"
+version = "46.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f9739b984380582bdb7749ae5b5d28839bce899212cf16465c1ac1f8b65d79"
+checksum = "1ad2cba786ae07da4d73371a88b9e0f9d3ffac1a9badc83922e0e15814f5c5fa"
 dependencies = [
  "ahash 0.8.3",
  "arrow-array",
@@ -4923,7 +5319,7 @@ dependencies = [
  "arrow-ipc",
  "arrow-schema",
  "arrow-select",
- "base64 0.21.2",
+ "base64 0.21.3",
  "brotli",
  "bytes",
  "chrono",
@@ -4963,9 +5359,9 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "regex-syntax 0.7.4",
+ "regex-syntax 0.7.5",
  "structmeta",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5023,6 +5419,12 @@ dependencies = [
  "prost-types",
 ]
 
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
 [[package]]
 name = "pem"
 version = "1.1.1"
@@ -5032,6 +5434,15 @@ dependencies = [
  "base64 0.13.1",
 ]
 
+[[package]]
+name = "pem-rfc7468"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f22eb0e3c593294a99e9ff4b24cf6b752d43f193aa4415fe5077c159996d497"
+dependencies = [
+ "base64ct",
+]
+
 [[package]]
 name = "pem-rfc7468"
 version = "0.7.0"
@@ -5145,14 +5556,14 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
+checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
 
 [[package]]
 name = "pin-utils"
@@ -5166,9 +5577,21 @@ version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
 dependencies = [
- "der",
- "pkcs8",
- "spki",
+ "der 0.7.8",
+ "pkcs8 0.10.2",
+ "spki 0.7.2",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447"
+dependencies = [
+ "der 0.4.5",
+ "pem-rfc7468 0.2.3",
+ "spki 0.4.1",
+ "zeroize",
 ]
 
 [[package]]
@@ -5177,8 +5600,8 @@ version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
 dependencies = [
- "der",
- "spki",
+ "der 0.7.8",
+ "spki 0.7.2",
 ]
 
 [[package]]
@@ -5260,7 +5683,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -5269,7 +5692,7 @@ version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.3",
  "byteorder",
  "bytes",
  "fallible-iterator",
@@ -5277,7 +5700,7 @@ dependencies = [
  "md-5",
  "memchr",
  "rand",
- "sha2",
+ "sha2 0.10.7",
  "stringprep",
 ]
 
@@ -5308,7 +5731,7 @@ dependencies = [
  "inferno",
  "libc",
  "log",
- "nix",
+ "nix 0.26.2",
  "once_cell",
  "parking_lot 0.12.1",
  "smallvec",
@@ -5389,6 +5812,16 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "priority-queue"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fff39edfcaec0d64e8d0da38564fad195d2d51b680940295fcc307366e101e61"
+dependencies = [
+ "autocfg",
+ "indexmap 1.9.3",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "0.1.5"
@@ -5499,7 +5932,7 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
- "time 0.3.27",
+ "time 0.3.28",
  "url",
 ]
 
@@ -5554,15 +5987,15 @@ version = "0.1.0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
  "workspace-hack",
 ]
 
 [[package]]
 name = "prost-reflect"
-version = "0.11.4"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "000e1e05ebf7b26e1eba298e66fe4eee6eb19c567d0ffb35e0dd34231cdac4c8"
+checksum = "6b823de344848e011658ac981009100818b322421676740546f8b52ed5249428"
 dependencies = [
  "once_cell",
  "prost",
@@ -5743,7 +6176,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
  "rand_chacha",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -5753,9 +6186,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+
 [[package]]
 name = "rand_core"
 version = "0.6.4"
@@ -5771,7 +6210,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
 dependencies = [
- "rand_core",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -5822,9 +6261,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.23.2"
+version = "0.23.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
+checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
 dependencies = [
  "combine",
  "itoa",
@@ -5855,14 +6294,14 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.3"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
+checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.3.6",
- "regex-syntax 0.7.4",
+ "regex-automata 0.3.8",
+ "regex-syntax 0.7.5",
 ]
 
 [[package]]
@@ -5876,13 +6315,13 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.6"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.4",
+ "regex-syntax 0.7.5",
 ]
 
 [[package]]
@@ -5893,9 +6332,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
 
 [[package]]
 name = "rend"
@@ -5914,7 +6353,7 @@ checksum = "3228e570df74d69d3d3236a71371f1edd748a3e4eb728ea1f29d403bc10fc727"
 dependencies = [
  "anyhow",
  "async-trait",
- "base64 0.21.2",
+ "base64 0.21.3",
  "chrono",
  "form_urlencoded",
  "hex",
@@ -5933,17 +6372,17 @@ dependencies = [
  "serde",
  "serde_json",
  "sha1",
- "sha2",
+ "sha2 0.10.7",
  "tokio",
 ]
 
 [[package]]
 name = "reqwest"
-version = "0.11.18"
+version = "0.11.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
+checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.3",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -5978,7 +6417,7 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots 0.22.6",
+ "webpki-roots 0.25.2",
  "winreg",
 ]
 
@@ -6012,7 +6451,7 @@ version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
- "clap",
+ "clap 4.4.2",
  "console",
  "fs-err",
  "glob",
@@ -6040,7 +6479,7 @@ name = "risedev-config"
 version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
- "clap",
+ "clap 4.4.2",
  "console",
  "dialoguer",
  "enum-iterator",
@@ -6072,7 +6511,7 @@ dependencies = [
 name = "risingwave_backup_cmd"
 version = "1.1.0-alpha"
 dependencies = [
- "clap",
+ "clap 4.4.2",
  "madsim-tokio",
  "prometheus",
  "risingwave_backup",
@@ -6135,13 +6574,13 @@ dependencies = [
  "bcc",
  "bytes",
  "bytesize",
- "clap",
+ "clap 4.4.2",
  "futures",
  "hdrhistogram",
  "itertools 0.11.0",
  "libc",
  "madsim-tokio",
- "nix",
+ "nix 0.26.2",
  "opentelemetry",
  "parking_lot 0.12.1",
  "prometheus",
@@ -6161,7 +6600,7 @@ dependencies = [
 name = "risingwave_cmd"
 version = "1.1.0-alpha"
 dependencies = [
- "clap",
+ "clap 4.4.2",
  "madsim-tokio",
  "prometheus",
  "risingwave_common",
@@ -6182,7 +6621,7 @@ name = "risingwave_cmd_all"
 version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
- "clap",
+ "clap 4.4.2",
  "console",
  "const-str",
  "expect-test",
@@ -6224,7 +6663,7 @@ dependencies = [
  "bytes",
  "chrono",
  "chrono-tz",
- "clap",
+ "clap 4.4.2",
  "comfy-table",
  "crc32fast",
  "criterion",
@@ -6239,7 +6678,9 @@ dependencies = [
  "futures",
  "hex",
  "http",
- "humantime",
+ "http-body",
+ "humantime 2.1.0",
+ "hyper",
  "hytra",
  "itertools 0.11.0",
  "itoa",
@@ -6257,6 +6698,7 @@ dependencies = [
  "parking_lot 0.12.1",
  "parse-display",
  "paste",
+ "pin-project-lite",
  "postgres-types",
  "pretty_assertions",
  "procfs 0.15.1",
@@ -6285,6 +6727,8 @@ dependencies = [
  "thiserror",
  "tinyvec",
  "toml 0.7.6",
+ "tower-layer",
+ "tower-service",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -6332,7 +6776,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "bytes",
- "clap",
+ "clap 4.4.2",
  "futures",
  "madsim-tokio",
  "prometheus",
@@ -6358,7 +6802,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "await-tree",
- "clap",
+ "clap 4.4.2",
  "madsim-tokio",
  "madsim-tonic",
  "parking_lot 0.12.1",
@@ -6383,7 +6827,7 @@ dependencies = [
  "async-trait",
  "await-tree",
  "chrono",
- "clap",
+ "clap 4.4.2",
  "either",
  "futures",
  "futures-async-stream",
@@ -6423,6 +6867,7 @@ dependencies = [
  "apache-avro 0.15.0 (git+https://github.com/risingwavelabs/avro?branch=idx0dev/resolved_schema)",
  "arrow-array",
  "arrow-schema",
+ "async-nats",
  "async-trait",
  "auto_enums",
  "auto_impl",
@@ -6432,7 +6877,7 @@ dependencies = [
  "aws-sdk-s3",
  "aws-smithy-http",
  "aws-types",
- "base64 0.21.2",
+ "base64 0.21.3",
  "bincode 1.3.3",
  "byteorder",
  "bytes",
@@ -6446,20 +6891,28 @@ dependencies = [
  "futures-async-stream",
  "glob",
  "google-cloud-pubsub",
+ "hyper",
+ "hyper-tls",
  "icelake",
  "itertools 0.11.0",
+ "jsonschema-transpiler",
  "madsim-rdkafka",
  "madsim-tokio",
  "madsim-tonic",
  "maplit",
  "moka",
+ "mysql_async",
+ "mysql_common",
  "nexmark",
  "num-bigint",
  "opendal",
  "parking_lot 0.12.1",
+ "paste",
  "prometheus",
  "prost",
+ "prost-build",
  "prost-reflect",
+ "prost-types",
  "protobuf-native",
  "pulsar",
  "rand",
@@ -6491,7 +6944,7 @@ dependencies = [
  "anyhow",
  "bytes",
  "chrono",
- "clap",
+ "clap 4.4.2",
  "comfy-table",
  "futures",
  "inquire",
@@ -6524,7 +6977,7 @@ version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
- "clap",
+ "clap 4.4.2",
  "madsim-tokio",
  "pg_interval",
  "rust_decimal",
@@ -6572,7 +7025,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha1",
- "sha2",
+ "sha2 0.10.7",
  "smallvec",
  "static_assertions",
  "thiserror",
@@ -6587,7 +7040,7 @@ dependencies = [
  "itertools 0.11.0",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -6603,7 +7056,7 @@ dependencies = [
  "auto_enums",
  "bk-tree",
  "bytes",
- "clap",
+ "clap 4.4.2",
  "downcast-rs",
  "dyn-clone",
  "easy-ext",
@@ -6645,7 +7098,7 @@ dependencies = [
  "risingwave_variables",
  "serde",
  "serde_json",
- "sha2",
+ "sha2 0.10.7",
  "smallvec",
  "tempfile",
  "thiserror",
@@ -6676,7 +7129,7 @@ version = "1.1.0-alpha"
 dependencies = [
  "async-trait",
  "bytes",
- "clap",
+ "clap 4.4.2",
  "criterion",
  "expect-test",
  "fail",
@@ -6727,6 +7180,19 @@ dependencies = [
 [[package]]
 name = "risingwave_java_binding"
 version = "0.1.0"
+dependencies = [
+ "prost",
+ "risingwave_common",
+ "risingwave_expr",
+ "risingwave_jni_core",
+ "risingwave_pb",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "risingwave_jni_core"
+version = "0.1.0"
 dependencies = [
  "bytes",
  "futures",
@@ -6758,7 +7224,7 @@ dependencies = [
  "aws-sdk-ec2",
  "axum",
  "bytes",
- "clap",
+ "clap 4.4.2",
  "crepe",
  "easy-ext",
  "either",
@@ -6881,7 +7347,7 @@ name = "risingwave_regress_test"
 version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
- "clap",
+ "clap 4.4.2",
  "madsim-tokio",
  "path-absolutize",
  "similar",
@@ -6939,7 +7405,7 @@ dependencies = [
  "prometheus",
  "risingwave_common",
  "risingwave_variables",
- "time 0.3.27",
+ "time 0.3.28",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -6952,7 +7418,8 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "clap",
+ "cfg-or-panic",
+ "clap 4.4.2",
  "console",
  "futures",
  "glob",
@@ -7047,7 +7514,7 @@ version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
- "clap",
+ "clap 4.4.2",
  "expect-test",
  "itertools 0.11.0",
  "libtest-mimic",
@@ -7072,7 +7539,7 @@ name = "risingwave_state_cleaning_test"
 version = "1.1.0-alpha"
 dependencies = [
  "anyhow",
- "clap",
+ "clap 4.4.2",
  "futures",
  "madsim-tokio",
  "prometheus",
@@ -7118,7 +7585,7 @@ dependencies = [
  "memcomparable",
  "moka",
  "more-asserts",
- "nix",
+ "nix 0.26.2",
  "num-integer",
  "parking_lot 0.12.1",
  "procfs 0.15.1",
@@ -7139,7 +7606,6 @@ dependencies = [
  "sled",
  "spin 0.9.8",
  "sync-point",
- "sysinfo",
  "tempfile",
  "thiserror",
  "tokio-retry",
@@ -7171,6 +7637,7 @@ dependencies = [
  "expect-test",
  "futures",
  "futures-async-stream",
+ "governor",
  "hytra",
  "iter-chunks",
  "itertools 0.11.0",
@@ -7183,6 +7650,7 @@ dependencies = [
  "multimap",
  "num-traits",
  "parking_lot 0.12.1",
+ "parse-display",
  "pin-project",
  "prometheus",
  "prost",
@@ -7285,17 +7753,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8"
 dependencies = [
  "byteorder",
- "const-oid",
- "digest",
+ "const-oid 0.9.5",
+ "digest 0.10.7",
  "num-bigint-dig",
  "num-integer",
  "num-iter",
  "num-traits",
  "pkcs1",
- "pkcs8",
- "rand_core",
- "signature",
- "spki",
+ "pkcs8 0.10.2",
+ "rand_core 0.6.4",
+ "signature 2.1.0",
+ "spki 0.7.2",
  "subtle",
  "zeroize",
 ]
@@ -7312,13 +7780,12 @@ dependencies = [
 
 [[package]]
 name = "rust_decimal"
-version = "1.31.0"
+version = "1.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea"
+checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd"
 dependencies = [
  "arrayvec",
  "borsh",
- "byteorder",
  "bytes",
  "num-traits",
  "postgres",
@@ -7334,6 +7801,12 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
@@ -7426,7 +7899,7 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.3",
 ]
 
 [[package]]
@@ -7495,6 +7968,12 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "saturating"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71"
+
 [[package]]
 name = "schannel"
 version = "0.1.22"
@@ -7605,9 +8084,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.185"
+version = "1.0.188"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31"
+checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
 dependencies = [
  "serde_derive",
 ]
@@ -7645,13 +8124,13 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.185"
+version = "1.0.188"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec"
+checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -7676,6 +8155,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_nanos"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ae801b7733ca8d6a2b580debe99f67f36826a0f5b8a36055dc6bc40f8d6bc71"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_path_to_error"
 version = "0.1.14"
@@ -7695,6 +8183,17 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_repr"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.31",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "0.6.3"
@@ -7739,7 +8238,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_with_macros 2.3.3",
- "time 0.3.27",
+ "time 0.3.28",
 ]
 
 [[package]]
@@ -7748,7 +8247,7 @@ version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.3",
  "chrono",
  "hex",
  "indexmap 1.9.3",
@@ -7756,7 +8255,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_with_macros 3.3.0",
- "time 0.3.27",
+ "time 0.3.28",
 ]
 
 [[package]]
@@ -7780,7 +8279,7 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -7792,7 +8291,7 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -7830,7 +8329,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -7841,7 +8340,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
 dependencies = [
  "cfg-if",
  "cpufeatures",
- "digest",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -7850,6 +8349,19 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
 
+[[package]]
+name = "sha2"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
+dependencies = [
+ "block-buffer 0.9.0",
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.9.0",
+ "opaque-debug",
+]
+
 [[package]]
 name = "sha2"
 version = "0.10.7"
@@ -7858,7 +8370,7 @@ checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
- "digest",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -7886,6 +8398,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
 
+[[package]]
+name = "shlex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+
 [[package]]
 name = "signal-hook"
 version = "0.3.17"
@@ -7916,14 +8434,32 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "signatory"
+version = "0.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfecc059e81632eef1dd9b79e22fc28b8fe69b30d3357512a77a0ad8ee3c782"
+dependencies = [
+ "pkcs8 0.7.6",
+ "rand_core 0.6.4",
+ "signature 1.6.4",
+ "zeroize",
+]
+
+[[package]]
+name = "signature"
+version = "1.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
+
 [[package]]
 name = "signature"
 version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500"
 dependencies = [
- "digest",
- "rand_core",
+ "digest 0.10.7",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -7961,7 +8497,7 @@ dependencies = [
  "num-bigint",
  "num-traits",
  "thiserror",
- "time 0.3.27",
+ "time 0.3.28",
 ]
 
 [[package]]
@@ -8085,6 +8621,15 @@ dependencies = [
  "lock_api",
 ]
 
+[[package]]
+name = "spki"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32"
+dependencies = [
+ "der 0.4.5",
+]
+
 [[package]]
 name = "spki"
 version = "0.7.2"
@@ -8092,7 +8637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a"
 dependencies = [
  "base64ct",
- "der",
+ "der 0.7.8",
 ]
 
 [[package]]
@@ -8106,7 +8651,7 @@ dependencies = [
  "fs-err",
  "futures",
  "glob",
- "humantime",
+ "humantime 2.1.0",
  "itertools 0.11.0",
  "libtest-mimic",
  "md-5",
@@ -8146,6 +8691,12 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "strsim"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -8161,7 +8712,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -8172,7 +8723,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -8213,7 +8764,17 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.29",
+ "syn 2.0.31",
+]
+
+[[package]]
+name = "subprocess"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086"
+dependencies = [
+ "libc",
+ "winapi",
 ]
 
 [[package]]
@@ -8258,9 +8819,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.29"
+version = "2.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
+checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8285,9 +8846,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
 [[package]]
 name = "sysinfo"
-version = "0.29.8"
+version = "0.29.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d10ed79c22663a35a255d289a7fdcb43559fc77ff15df5ce6c341809e7867528"
+checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -8345,24 +8906,33 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
+[[package]]
+name = "textwrap"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "thiserror"
-version = "1.0.47"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
+checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.47"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
+checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -8447,9 +9017,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.27"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07"
+checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48"
 dependencies = [
  "deranged",
  "itoa",
@@ -8468,9 +9038,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "time-macros"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9"
+checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572"
 dependencies = [
  "time-core",
 ]
@@ -8547,7 +9117,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -8714,7 +9284,7 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum",
- "base64 0.21.2",
+ "base64 0.21.3",
  "bytes",
  "flate2",
  "futures-core",
@@ -8773,9 +9343,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82"
+checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
 dependencies = [
  "bitflags 2.4.0",
  "bytes",
@@ -8829,7 +9399,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -8867,12 +9437,14 @@ dependencies = [
 
 [[package]]
 name = "tracing-opentelemetry"
-version = "0.20.0"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc09e402904a5261e42cf27aea09ccb7d5318c6717a9eec3d8e2e65c56b18f19"
+checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8"
 dependencies = [
  "once_cell",
  "opentelemetry",
+ "opentelemetry_sdk",
+ "smallvec",
  "tracing",
  "tracing-core",
  "tracing-log",
@@ -8905,7 +9477,7 @@ dependencies = [
  "sharded-slab",
  "smallvec",
  "thread_local",
- "time 0.3.27",
+ "time 0.3.28",
  "tracing",
  "tracing-core",
  "tracing-log",
@@ -9046,9 +9618,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
 [[package]]
 name = "url"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
+checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -9103,6 +9675,12 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
 [[package]]
 name = "vergen"
 version = "8.2.4"
@@ -9111,7 +9689,7 @@ checksum = "bbc5ad0d9d26b2c49a5ab7da76c3e79d3ee37e7821799f8223fcb8f2f391a2e7"
 dependencies = [
  "anyhow",
  "rustversion",
- "time 0.3.27",
+ "time 0.3.28",
 ]
 
 [[package]]
@@ -9149,9 +9727,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
 
 [[package]]
 name = "walkdir"
-version = "2.3.3"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
+checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
 dependencies = [
  "same-file",
  "winapi-util",
@@ -9199,7 +9777,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
  "wasm-bindgen-shared",
 ]
 
@@ -9233,7 +9811,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -9246,9 +9824,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "wasm-streams"
-version = "0.2.3"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
+checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
 dependencies = [
  "futures-util",
  "js-sys",
@@ -9279,21 +9857,18 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.22.6"
+version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
+checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338"
 dependencies = [
- "webpki",
+ "rustls-webpki 0.100.2",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "0.23.1"
+version = "0.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338"
-dependencies = [
- "rustls-webpki 0.100.2",
-]
+checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
 
 [[package]]
 name = "which"
@@ -9489,11 +10064,12 @@ dependencies = [
 
 [[package]]
 name = "winreg"
-version = "0.10.1"
+version = "0.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
 dependencies = [
- "winapi",
+ "cfg-if",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -9517,17 +10093,21 @@ dependencies = [
  "aws-credential-types",
  "aws-sdk-s3",
  "aws-smithy-client",
+ "byteorder",
  "bytes",
  "cc",
  "chrono",
- "clap",
+ "clap 4.4.2",
  "clap_builder",
  "combine",
  "crossbeam-epoch",
  "crossbeam-utils",
+ "deranged",
  "either",
  "fail",
  "fixedbitset",
+ "flate2",
+ "frunk_core",
  "futures",
  "futures-channel",
  "futures-core",
@@ -9553,8 +10133,9 @@ dependencies = [
  "madsim-tokio",
  "memchr",
  "miniz_oxide",
+ "mio",
  "multimap",
- "nix",
+ "nom",
  "num-bigint",
  "num-integer",
  "num-traits",
@@ -9571,10 +10152,10 @@ dependencies = [
  "prost",
  "rand",
  "rand_chacha",
- "rand_core",
+ "rand_core 0.6.4",
  "regex",
- "regex-automata 0.3.6",
- "regex-syntax 0.7.4",
+ "regex-automata 0.3.8",
+ "regex-syntax 0.7.5",
  "reqwest",
  "ring",
  "rust_decimal",
@@ -9585,8 +10166,9 @@ dependencies = [
  "smallvec",
  "subtle",
  "syn 1.0.109",
- "syn 2.0.29",
- "time 0.3.27",
+ "syn 2.0.31",
+ "time 0.3.28",
+ "time-macros",
  "tinyvec",
  "tokio",
  "tokio-postgres",
@@ -9601,6 +10183,7 @@ dependencies = [
  "unicode-normalization",
  "url",
  "uuid",
+ "zeroize",
 ]
 
 [[package]]
@@ -9630,9 +10213,9 @@ dependencies = [
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"
+checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b"
 
 [[package]]
 name = "xz2"
@@ -9676,7 +10259,7 @@ checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.31",
 ]
 
 [[package]]
@@ -9684,6 +10267,20 @@ name = "zeroize"
 version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
+dependencies = [
+ "zeroize_derive",
+]
+
+[[package]]
+name = "zeroize_derive"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.31",
+]
 
 [[package]]
 name = "zstd"
diff --git a/Cargo.toml b/Cargo.toml
index 0dcd9e05ade9a..0c56df58c5e24 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,6 +15,7 @@ members = [
   "src/frontend",
   "src/frontend/planner_test",
   "src/java_binding",
+  "src/jni_core",
   "src/meta",
   "src/object_store",
   "src/prost",
@@ -74,12 +75,17 @@ aws-types = "0.55"
 etcd-client = { package = "madsim-etcd-client", version = "0.3" }
 futures-async-stream = "0.2"
 hytra = "0.1"
-rdkafka = { package = "madsim-rdkafka", git = "https://github.com/madsim-rs/madsim.git", rev = "f01e3ca", features = ["cmake-build"] }
+rdkafka = { package = "madsim-rdkafka", git = "https://github.com/madsim-rs/madsim.git", rev = "bb8f063", features = ["cmake-build"] }
 hashbrown = { version = "0.14.0", features = ["ahash", "inline-more", "nightly"] }
 criterion = { version = "0.5", features = ["async_futures"] }
 tonic = { package = "madsim-tonic", version = "0.3.1" }
 tonic-build = { package = "madsim-tonic-build", version = "0.3.1" }
-icelake = { git = "https://github.com/icelake-io/icelake", rev = "ec7d251e5e6db0673cea35edd62e6c7df81c4759" }
+icelake = { git = "https://github.com/icelake-io/icelake", rev = "a6790d17094754959e351fac1e11147e37643e97" }
+arrow-array = "46"
+arrow-schema = "46"
+arrow-buffer = "46"
+arrow-flight = "46"
+arrow-select = "46"
 
 risingwave_backup = { path = "./src/storage/backup" }
 risingwave_batch = { path = "./src/batch" }
@@ -108,6 +114,8 @@ risingwave_stream = { path = "./src/stream" }
 risingwave_test_runner = { path = "./src/test_runner" }
 risingwave_udf = { path = "./src/udf" }
 risingwave_variables = { path = "./src/utils/variables" }
+risingwave_java_binding = { path = "./src/java_binding" }
+risingwave_jni_core = { path = "src/jni_core" }
 
 [profile.dev]
 lto = 'off'
diff --git a/Makefile.toml b/Makefile.toml
index 1faf715e984a7..42ce20c0769ed 100644
--- a/Makefile.toml
+++ b/Makefile.toml
@@ -579,7 +579,53 @@ description = "Kill RisingWave dev cluster"
 script = '''
 #!/usr/bin/env bash
 
-tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+set -euo pipefail
+
+wait_kafka_exit() {
+  # Follow kafka-server-stop.sh
+  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for kafka to exit"
+    sleep 1
+  done
+}
+
+wait_zookeeper_exit() {
+  # Follow zookeeper-server-stop.sh
+  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for zookeeper to exit"
+    sleep 1
+  done
+}
+
+kill_kafka() {
+  ${PREFIX_BIN}/kafka/bin/kafka-server-stop.sh
+  wait_kafka_exit
+}
+
+kill_zookeeper() {
+  ${PREFIX_BIN}/kafka/bin/zookeeper-server-stop.sh
+  wait_zookeeper_exit
+}
+
+# Kill other components
+tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
+| grep -v 'kafka' \
+| grep -v 'zookeeper' \
+| awk '{ print $2 }' \
+| xargs -I {} tmux send-keys -t {} C-c C-d
+
+if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
+then
+  echo "kill kafka"
+  kill_kafka
+
+  echo "kill zookeeper"
+  kill_zookeeper
+
+  # Kill their tmux sessions
+  tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+fi
+
 tmux kill-session -t risedev
 test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
 '''
@@ -671,7 +717,7 @@ script = """
 set -e
 
 export CARGO_TARGET_DIR=target/coverage
-cargo llvm-cov nextest --html
+cargo llvm-cov nextest --html --workspace --exclude risingwave_simulation
 """
 description = "Run unit tests and report coverage"
 
@@ -682,7 +728,7 @@ script = """
 #!/usr/bin/env bash
 set -e
 
-cargo nextest run "$@"
+cargo nextest run --workspace --exclude risingwave_simulation "$@"
 """
 description = "🌟 Run unit tests"
 
@@ -1238,3 +1284,8 @@ cat << EOF > src/config/example.toml
 EOF
 cargo run -p risingwave_common --bin example-config >> src/config/example.toml
 '''
+
+[tasks.backwards-compat-test]
+category = "RiseDev - Backwards Compatibility Test"
+description = "Run backwards compatibility test"
+script = "./backwards-compat-tests/scripts/run_local.sh"
\ No newline at end of file
diff --git a/README.md b/README.md
index 9daa030431951..d63369b0200a6 100644
--- a/README.md
+++ b/README.md
@@ -18,14 +18,14 @@ RisingWave ingests data from sources like Apache Kafka, Apache Pulsar, Amazon Ki
 
 RisingWave 1.0 is a battle-tested version that has undergone rigorous stress tests and performance evaluations. It has proven its reliability and efficiency through successful deployments in numerous production environments across dozens of companies.
 
-Learn more at [Introduction to RisingWave](https://www.risingwave.dev/docs/current/intro/).
+Learn more at [Introduction to RisingWave](https://docs.risingwave.com/docs/current/intro/).
 
 ![RisingWave](https://github.com/risingwavelabs/risingwave-docs/blob/0f7e1302b22493ba3c1c48e78810750ce9a5ff42/docs/images/archi_simple.png)
 
 
 ## RisingWave Cloud
 
-RisingWave Cloud is a fully-managed and scalable stream processing platform powered by the open-source RisingWave project. Try it out for free at: [risingwave.com/cloud](https://risingwave.com/cloud).
+RisingWave Cloud is a fully-managed and scalable stream processing platform powered by the open-source RisingWave project. Try it out for free at: [cloud.risingwave.com](https://cloud.risingwave.com).
 
 ## Notes on telemetry
 
@@ -33,15 +33,15 @@ RisingWave collects anonymous usage statistics to better understand how the comm
 
 ## Get started
 
-- To learn about how to install and run RisingWave, see [Get started](https://www.risingwave.dev/docs/current/get-started/).
-- To learn about how to ingest data and the supported data sources, see [Sources](https://www.risingwave.dev/docs/current/data-ingestion/).
-- To learn about how to transform data using the PostgreSQL-compatible SQL of RisingWave, see [SQL reference](https://www.risingwave.dev/docs/current/sql-references/).
-- To learn about how to deliver data and the supported data sinks, see [Sinks](https://www.risingwave.dev/docs/current/data-delivery/).
-- To learn about new features and changes in the current and previous versions, see [Release notes](https://www.risingwave.dev/docs/current/release-notes/).
+- To learn about how to install and run RisingWave, see [Get started](https://docs.risingwave.com/docs/current/get-started/).
+- To learn about how to ingest data and the supported data sources, see [Sources](https://docs.risingwave.com/docs/current/data-ingestion/).
+- To learn about how to transform data using the PostgreSQL-compatible SQL of RisingWave, see [SQL reference](https://docs.risingwave.com/docs/current/sql-references/).
+- To learn about how to deliver data and the supported data sinks, see [Sinks](https://docs.risingwave.com/docs/current/data-delivery/).
+- To learn about new features and changes in the current and previous versions, see [Release notes](https://docs.risingwave.com/release-notes/).
 
 ## Documentation
 
-To learn about how to use RisingWave, refer to [RisingWave User Documentation](https://www.risingwave.dev/). To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [readme.md](docs/README.md).
+To learn about how to use RisingWave, refer to [RisingWave User Documentation](https://docs.risingwave.com/). To learn about the development process, see the [developer guide](docs/developer-guide.md). To understand the design and implementation of RisingWave, refer to the design docs listed in [readme.md](docs/README.md).
 
 ## Community
 
diff --git a/backwards-compat-tests/README.md b/backwards-compat-tests/README.md
new file mode 100644
index 0000000000000..b5036d3ce39f7
--- /dev/null
+++ b/backwards-compat-tests/README.md
@@ -0,0 +1,18 @@
+# Backwards Compatibility Tests
+
+The backwards compatibility tests run in the following manner:
+1. Prepare old-cluster artifacts
+2. Configure the old-cluster.
+3. Start the old-cluster.
+4. Run DDL / DML / DQL.
+5. Stop the old-cluster.
+6. Prepare new-cluster artifacts.
+7. Configure the new-cluster.
+8. Start the new-cluster.
+9. Verify results of step 4.
+
+We currently cover the following:
+1. Basic mv
+2. Nexmark (on rw table not nexmark source)
+3. TPC-H
+4. Kafka Source
\ No newline at end of file
diff --git a/backwards-compat-tests/scripts/run_local.sh b/backwards-compat-tests/scripts/run_local.sh
new file mode 100755
index 0000000000000..cfa164ec35a29
--- /dev/null
+++ b/backwards-compat-tests/scripts/run_local.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ORIGINAL_BRANCH=$(git branch --show-current)
+
+on_exit() {
+  git checkout "$ORIGINAL_BRANCH"
+}
+
+trap on_exit EXIT
+
+source backwards-compat-tests/scripts/utils.sh
+
+configure_rw() {
+echo "--- Setting up cluster config"
+cat <<EOF > risedev-profiles.user.yml
+full-without-monitoring:
+  steps:
+    - use: minio
+    - use: etcd
+    - use: meta-node
+    - use: compute-node
+    - use: frontend
+    - use: compactor
+    - use: zookeeper
+    - use: kafka
+EOF
+
+cat <<EOF > risedev-components.user.env
+RISEDEV_CONFIGURED=false
+
+ENABLE_MINIO=true
+ENABLE_ETCD=true
+ENABLE_KAFKA=true
+
+# Fetch risingwave binary from release.
+ENABLE_BUILD_RUST=true
+
+# Ensure it will link the all-in-one binary from our release.
+ENABLE_ALL_IN_ONE=true
+
+# ENABLE_RELEASE_PROFILE=true
+EOF
+}
+
+setup_old_cluster() {
+  echo "--- Setting up old cluster"
+  git checkout "v${OLD_VERSION}-rc"
+}
+
+setup_new_cluster() {
+  echo "--- Setting up new cluster"
+  rm -r .risingwave/bin/risingwave
+  git checkout main
+}
+
+main() {
+  set -euo pipefail
+  get_rw_versions
+  setup_old_cluster
+  configure_rw
+  seed_old_cluster "$OLD_VERSION"
+
+  setup_new_cluster
+  configure_rw
+  validate_new_cluster "$NEW_VERSION"
+}
+
+main
\ No newline at end of file
diff --git a/backwards-compat-tests/scripts/utils.sh b/backwards-compat-tests/scripts/utils.sh
new file mode 100644
index 0000000000000..e047b9f44f421
--- /dev/null
+++ b/backwards-compat-tests/scripts/utils.sh
@@ -0,0 +1,243 @@
+#!/usr/bin/env bash
+
+# NOTE(kwannoel):
+# Do not run this script directly, it is meant to be sourced.
+# Backwards compatibility tests consist of the following parts:
+#
+# 1. Setup old cluster binaries.
+# 2. Seed old cluster.
+# 3. Setup new cluster binaries.
+# 4. Run validation on new cluster.
+#
+# Steps 1,3 are specific to the execution environment, CI / Local.
+# This script only provides utilities for 2, 4.
+
+################################### ENVIRONMENT CONFIG
+
+# Duration to wait for recovery (seconds)
+RECOVERY_DURATION=20
+
+# Setup test directory
+TEST_DIR=.risingwave/backwards-compat-tests/
+KAFKA_PATH=.risingwave/bin/kafka
+mkdir -p $TEST_DIR
+cp -r backwards-compat-tests/slt/* $TEST_DIR
+
+wait_kafka_exit() {
+  # Follow kafka-server-stop.sh
+  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for kafka to exit"
+    sleep 1
+  done
+}
+
+wait_zookeeper_exit() {
+  # Follow zookeeper-server-stop.sh
+  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
+    echo "Waiting for zookeeper to exit"
+    sleep 1
+  done
+}
+
+kill_kafka() {
+  $KAFKA_PATH/bin/kafka-server-stop.sh
+  wait_kafka_exit
+}
+
+kill_zookeeper() {
+  $KAFKA_PATH/bin/zookeeper-server-stop.sh
+  wait_zookeeper_exit
+}
+
+# Older versions of RW may not gracefully kill kafka.
+# So we duplicate the definition here.
+kill_cluster() {
+  # Kill other components
+  tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
+  | grep -v 'kafka' \
+  | grep -v 'zookeeper' \
+  | awk '{ print $2 }' \
+  | xargs -I {} tmux send-keys -t {} C-c C-d
+
+  set +e
+  if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
+  then
+    echo "kill kafka"
+    kill_kafka
+
+    echo "kill zookeeper"
+    kill_zookeeper
+
+    # Kill their tmux sessions
+    tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+  fi
+  set -e
+
+  tmux kill-session -t risedev
+  test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
+}
+
+run_sql () {
+  psql -h localhost -p 4566 -d dev -U root -c "$@"
+}
+
+check_version() {
+  local VERSION=$1
+  local raw_version=$(run_sql "SELECT version();")
+  echo "--- Version"
+  echo "$raw_version"
+  local version=$(echo $raw_version | grep -i risingwave | sed 's/^.*risingwave-\([0-9]*\.[0-9]*\.[0-9]\).*$/\1/i')
+  if [[ "$version" != "$VERSION" ]]; then
+    echo "Version mismatch, expected $VERSION, got $version"
+    exit 1
+  fi
+}
+
+create_kafka_topic() {
+  "$KAFKA_PATH"/bin/kafka-topics.sh \
+    --create \
+    --topic backwards_compat_test_kafka_source --bootstrap-server localhost:29092
+}
+
+insert_json_kafka() {
+  local JSON=$1
+  echo "$JSON" | "$KAFKA_PATH"/bin/kafka-console-producer.sh \
+    --topic backwards_compat_test_kafka_source \
+    --bootstrap-server localhost:29092
+}
+
+seed_json_kafka() {
+  insert_json_kafka '{"timestamp": "2023-07-28 07:11:00", "user_id": 1, "page_id": 1, "action": "gtrgretrg"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 07:11:00", "user_id": 2, "page_id": 1, "action": "fsdfgerrg"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 07:11:00", "user_id": 3, "page_id": 1, "action": "sdfergtth"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 4, "page_id": 2, "action": "erwerhghj"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 5, "page_id": 2, "action": "kiku7ikkk"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 6, "page_id": 3, "action": "6786745ge"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 7, "page_id": 3, "action": "fgbgfnyyy"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 8, "page_id": 4, "action": "werwerwwe"}'
+  insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 9, "page_id": 4, "action": "yjtyjtyyy"}'
+}
+
+################################### Entry Points
+
+# Get $OLD_VERSION and $NEW_VERSION for Risingwave
+get_rw_versions() {
+  # For backwards compat test we assume we are testing the latest version of RW (i.e. latest main commit)
+  # against the Nth latest release candidate, where N > 1. N can be larger,
+  # in case some old cluster did not upgrade.
+  local VERSION_OFFSET=4
+
+  # First we obtain a list of versions from git branch names.
+  # Then we normalize them to semver format (MAJOR.MINOR.PATCH).
+  echo "--- git branch origin output"
+  git branch -r | grep origin
+
+  echo "--- VERSION BRANCHES"
+  local branches=$(git branch -r | grep -E "^  origin\/v[0-9]*\.[0-9]*.*-rc" | tr -d ' ' | sed -E 's/origin\/v([0-9]*\.[0-9])\-rc/\1.0/' | tr -d '\-vrcorigin\/' | tr -d ' ')
+  echo "$branches"
+
+  # Then we sort them in descending order.
+  echo "--- VERSIONS"
+  local sorted_versions=$(echo -e "$branches" | sort -t '.' -n)
+  echo "$sorted_versions"
+
+  # Then we take the Nth latest version.
+  # We set $OLD_VERSION to this.
+  OLD_VERSION=$(echo -e "$sorted_versions" | tail -n $VERSION_OFFSET | head -1)
+
+  # Next, for $NEW_VERSION we just scrape it from `workspace.package.version`.
+  NEW_VERSION=$(cat Cargo.toml | grep "\[workspace\.package\]" -A 5 | sed -n 's/version = \"\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p' | tr -d ' ')
+
+  # Then we assert that `$OLD_VERSION` < `$NEW_VERSION`.
+  local TOP=$(echo -e "$OLD_VERSION\n$NEW_VERSION" | sort -t '.' -n | tail -1)
+  if [[ "$TOP" != "$NEW_VERSION" ]]
+  then
+    echo "ERROR: $OLD_VERSION > $NEW_VERSION"
+    exit 1
+  else
+    echo "OLD_VERSION: $OLD_VERSION"
+    echo "NEW_VERSION: $NEW_VERSION"
+  fi
+}
+
+# Setup table and materialized view.
+# Run updates and deletes on the table.
+# Get the results.
+# TODO: Run nexmark, tpch queries
+# TODO(kwannoel): use sqllogictest.
+seed_old_cluster() {
+  # Caller should make sure the test env has these.
+  # They are called here because the current tests
+  # may not be backwards compatible, so we need to call
+  # them in old cluster environment.
+  cp -r e2e_test/streaming/nexmark $TEST_DIR
+  cp -r e2e_test/nexmark/* $TEST_DIR/nexmark
+
+  cp -r e2e_test/batch/tpch $TEST_DIR
+  cp -r e2e_test/tpch/* $TEST_DIR/tpch
+
+  ./risedev clean-data
+  ./risedev d full-without-monitoring && rm .risingwave/log/*
+
+  check_version "$OLD_VERSION"
+
+  echo "--- BASIC TEST: Seeding old cluster with data"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/basic/seed.slt"
+
+  echo "--- BASIC TEST: Validating old cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/basic/validate_original.slt"
+
+  echo "--- NEXMARK TEST: Seeding old cluster with data"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/nexmark-backwards-compat/seed.slt"
+
+  echo "--- NEXMARK TEST: Validating old cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/nexmark-backwards-compat/validate_original.slt"
+
+  echo "--- TPCH TEST: Seeding old cluster with data"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/tpch-backwards-compat/seed.slt"
+
+  echo "--- TPCH TEST: Validating old cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/tpch-backwards-compat/validate_original.slt"
+
+  echo "--- KAFKA TEST: Seeding old cluster with data"
+  create_kafka_topic
+  seed_json_kafka
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/seed.slt"
+
+  echo "--- KAFKA TEST: wait 5s for kafka to process data"
+  sleep 5
+
+  echo "--- KAFKA TEST: Validating old cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/validate_original.slt"
+
+  echo "--- Killing cluster"
+  kill_cluster
+  echo "--- Killed cluster"
+}
+
+validate_new_cluster() {
+  echo "--- Start cluster on latest"
+  ./risedev d full-without-monitoring
+
+  echo "--- Wait ${RECOVERY_DURATION}s for Recovery on Old Cluster Data"
+  sleep $RECOVERY_DURATION
+
+  check_version "$NEW_VERSION"
+
+  echo "--- BASIC TEST: Validating new cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/basic/validate_restart.slt"
+
+  echo "--- NEXMARK TEST: Validating new cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/nexmark-backwards-compat/validate_restart.slt"
+
+  echo "--- TPCH TEST: Validating new cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/tpch-backwards-compat/validate_restart.slt"
+
+  echo "--- KAFKA TEST: Seeding new cluster with data"
+  seed_json_kafka
+
+  echo "--- KAFKA TEST: Validating new cluster"
+  sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/validate_restart.slt"
+
+  kill_cluster
+}
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/basic/seed.slt b/backwards-compat-tests/slt/basic/seed.slt
new file mode 100644
index 0000000000000..57b19a4885e7f
--- /dev/null
+++ b/backwards-compat-tests/slt/basic/seed.slt
@@ -0,0 +1,14 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+CREATE TABLE t(v1 int primary key, v2 int);
+
+statement ok
+INSERT INTO t SELECT a AS v1, a * 2 AS v2 FROM generate_series(1, 10000) AS s(a);
+
+statement ok
+CREATE MATERIALIZED VIEW m as SELECT * from t;
+
+statement ok
+UPDATE t SET v2 = v2 + 1 WHERE v1 >= 1 AND v1 <= 5000;
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/basic/validate_original.slt b/backwards-compat-tests/slt/basic/validate_original.slt
new file mode 100644
index 0000000000000..7df2a71b1eb71
--- /dev/null
+++ b/backwards-compat-tests/slt/basic/validate_original.slt
@@ -0,0 +1,19 @@
+query I
+SELECT count(*) FROM t;
+----
+10000
+
+query I
+SELECT sum(v1), sum(v2) FROM t;
+----
+50005000 100015000
+
+query I
+SELECT count(*) FROM m;
+----
+10000
+
+query I
+SELECT sum(v1), sum(v2) FROM m;
+----
+50005000 100015000
diff --git a/backwards-compat-tests/slt/basic/validate_restart.slt b/backwards-compat-tests/slt/basic/validate_restart.slt
new file mode 100644
index 0000000000000..1a731813855a9
--- /dev/null
+++ b/backwards-compat-tests/slt/basic/validate_restart.slt
@@ -0,0 +1,41 @@
+# Rerun Original validation
+query I
+SELECT count(*) FROM t;
+----
+10000
+
+query I
+SELECT sum(v1), sum(v2) FROM t;
+----
+50005000 100015000
+
+query I
+SELECT count(*) FROM m;
+----
+10000
+
+query I
+SELECT sum(v1), sum(v2) FROM m;
+----
+50005000 100015000
+
+# Test updates and deletes
+
+statement ok
+SET RW_IMPLICIT_FLUSH=true;
+
+statement ok
+UPDATE t SET v2 = v2 + 1 WHERE v1 >= 1 AND v1 <= 5000;
+
+statement ok
+DELETE FROM t WHERE v1 > 5000;
+
+query I
+SELECT COUNT(*) FROM t;
+----
+5000
+
+query I
+SELECT SUM(v2) FROM t;
+----
+25015000
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/kafka/seed.slt b/backwards-compat-tests/slt/kafka/seed.slt
new file mode 100644
index 0000000000000..3840ce0c96b15
--- /dev/null
+++ b/backwards-compat-tests/slt/kafka/seed.slt
@@ -0,0 +1,19 @@
+statement ok
+CREATE SOURCE IF NOT EXISTS kafka_source
+(
+ action varchar,
+ user_id integer,
+ obj_id integer,
+ name varchar,
+ page_id integer,
+ age integer
+)
+WITH (
+ connector='kafka',
+ topic='backwards_compat_test_kafka_source',
+ properties.bootstrap.server='localhost:29092',
+ scan.startup.mode='earliest',
+) FORMAT PLAIN ENCODE JSON;
+
+statement ok
+CREATE MATERIALIZED VIEW kafka_mv1 as SELECT * FROM kafka_source;
diff --git a/backwards-compat-tests/slt/kafka/validate_original.slt b/backwards-compat-tests/slt/kafka/validate_original.slt
new file mode 100644
index 0000000000000..02fd973c25fff
--- /dev/null
+++ b/backwards-compat-tests/slt/kafka/validate_original.slt
@@ -0,0 +1,12 @@
+query I rowsort
+SELECT * FROM kafka_mv1;
+----
+6786745ge 6 NULL NULL 3 NULL
+erwerhghj 4 NULL NULL 2 NULL
+fgbgfnyyy 7 NULL NULL 3 NULL
+fsdfgerrg 2 NULL NULL 1 NULL
+gtrgretrg 1 NULL NULL 1 NULL
+kiku7ikkk 5 NULL NULL 2 NULL
+sdfergtth 3 NULL NULL 1 NULL
+werwerwwe 8 NULL NULL 4 NULL
+yjtyjtyyy 9 NULL NULL 4 NULL
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/kafka/validate_restart.slt b/backwards-compat-tests/slt/kafka/validate_restart.slt
new file mode 100644
index 0000000000000..7058b118f4d20
--- /dev/null
+++ b/backwards-compat-tests/slt/kafka/validate_restart.slt
@@ -0,0 +1,52 @@
+# create a new mv on source, it should retrieve all records
+# and match mv1.
+statement ok
+CREATE MATERIALIZED VIEW kafka_mv2 as SELECT * FROM kafka_source;
+
+sleep 5s
+
+query I rowsort
+SELECT * FROM kafka_mv2;
+----
+6786745ge 6 NULL NULL 3 NULL
+6786745ge 6 NULL NULL 3 NULL
+erwerhghj 4 NULL NULL 2 NULL
+erwerhghj 4 NULL NULL 2 NULL
+fgbgfnyyy 7 NULL NULL 3 NULL
+fgbgfnyyy 7 NULL NULL 3 NULL
+fsdfgerrg 2 NULL NULL 1 NULL
+fsdfgerrg 2 NULL NULL 1 NULL
+gtrgretrg 1 NULL NULL 1 NULL
+gtrgretrg 1 NULL NULL 1 NULL
+kiku7ikkk 5 NULL NULL 2 NULL
+kiku7ikkk 5 NULL NULL 2 NULL
+sdfergtth 3 NULL NULL 1 NULL
+sdfergtth 3 NULL NULL 1 NULL
+werwerwwe 8 NULL NULL 4 NULL
+werwerwwe 8 NULL NULL 4 NULL
+yjtyjtyyy 9 NULL NULL 4 NULL
+yjtyjtyyy 9 NULL NULL 4 NULL
+
+# MV1 should also have new records
+query I rowsort
+SELECT * FROM kafka_mv1;
+----
+6786745ge 6 NULL NULL 3 NULL
+6786745ge 6 NULL NULL 3 NULL
+erwerhghj 4 NULL NULL 2 NULL
+erwerhghj 4 NULL NULL 2 NULL
+fgbgfnyyy 7 NULL NULL 3 NULL
+fgbgfnyyy 7 NULL NULL 3 NULL
+fsdfgerrg 2 NULL NULL 1 NULL
+fsdfgerrg 2 NULL NULL 1 NULL
+gtrgretrg 1 NULL NULL 1 NULL
+gtrgretrg 1 NULL NULL 1 NULL
+kiku7ikkk 5 NULL NULL 2 NULL
+kiku7ikkk 5 NULL NULL 2 NULL
+sdfergtth 3 NULL NULL 1 NULL
+sdfergtth 3 NULL NULL 1 NULL
+werwerwwe 8 NULL NULL 4 NULL
+werwerwwe 8 NULL NULL 4 NULL
+yjtyjtyyy 9 NULL NULL 4 NULL
+yjtyjtyyy 9 NULL NULL 4 NULL
+
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt b/backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt
new file mode 100644
index 0000000000000..09fc6387ad65f
--- /dev/null
+++ b/backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt
@@ -0,0 +1,8 @@
+statement ok
+DELETE FROM person;
+
+statement ok
+DELETE FROM auction;
+
+statement ok
+DELETE FROM bid;
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt b/backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt
new file mode 100644
index 0000000000000..27201dcc49a21
--- /dev/null
+++ b/backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt
@@ -0,0 +1,3 @@
+include ../nexmark/insert_person.slt.part
+include ../nexmark/insert_auction.slt.part
+include ../nexmark/insert_bid.slt.part
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt b/backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt
new file mode 100644
index 0000000000000..bb86131e6a71a
--- /dev/null
+++ b/backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt
@@ -0,0 +1,9 @@
+include ../nexmark/create_tables.slt.part
+
+# First, insert the data into the tables
+include ../nexmark/insert_person.slt.part
+include ../nexmark/insert_auction.slt.part
+include ../nexmark/insert_bid.slt.part
+
+# Then, create materialized views based on the historical data (snapshot)
+include ../nexmark/create_views.slt.part
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt b/backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt
new file mode 100644
index 0000000000000..7e4dcdc800ee9
--- /dev/null
+++ b/backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt
@@ -0,0 +1 @@
+include ../nexmark/test_mv_result.slt.part
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt b/backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt
new file mode 100644
index 0000000000000..093f88f4fdbbe
--- /dev/null
+++ b/backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt
@@ -0,0 +1,11 @@
+include ../nexmark/test_mv_result.slt.part
+
+include ./delete.slt.part
+include ../nexmark/insert_person.slt.part
+include ../nexmark/insert_auction.slt.part
+include ../nexmark/insert_bid.slt.part
+include ../nexmark/test_mv_result.slt.part
+
+include ../nexmark/drop_views.slt.part
+include ../nexmark/drop_tables.slt.part
+
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/delete.slt b/backwards-compat-tests/slt/tpch-backwards-compat/delete.slt
new file mode 100644
index 0000000000000..e2a9b5e7b32c2
--- /dev/null
+++ b/backwards-compat-tests/slt/tpch-backwards-compat/delete.slt
@@ -0,0 +1,23 @@
+statement ok
+DELETE FROM supplier;
+
+statement ok
+DELETE FROM part;
+
+statement ok
+DELETE FROM partsupp;
+
+statement ok
+DELETE FROM customer;
+
+statement ok
+DELETE FROM orders;
+
+statement ok
+DELETE FROM lineitem;
+
+statement ok
+DELETE FROM nation;
+
+statement ok
+DELETE FROM region;
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/insert.slt b/backwards-compat-tests/slt/tpch-backwards-compat/insert.slt
new file mode 100644
index 0000000000000..c87003aab1e4b
--- /dev/null
+++ b/backwards-compat-tests/slt/tpch-backwards-compat/insert.slt
@@ -0,0 +1,8 @@
+include ../tpch/insert_customer.slt.part
+include ../tpch/insert_lineitem.slt.part
+include ../tpch/insert_nation.slt.part
+include ../tpch/insert_orders.slt.part
+include ../tpch/insert_part.slt.part
+include ../tpch/insert_partsupp.slt.part
+include ../tpch/insert_supplier.slt.part
+include ../tpch/insert_region.slt.part
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/seed.slt b/backwards-compat-tests/slt/tpch-backwards-compat/seed.slt
new file mode 100644
index 0000000000000..b56cefeab247e
--- /dev/null
+++ b/backwards-compat-tests/slt/tpch-backwards-compat/seed.slt
@@ -0,0 +1,19 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+SET QUERY_MODE TO distributed;
+
+statement ok
+SET CREATE_COMPACTION_GROUP_FOR_MV TO true;
+
+include ../tpch/create_tables.slt.part
+
+include ../tpch/insert_customer.slt.part
+include ../tpch/insert_lineitem.slt.part
+include ../tpch/insert_nation.slt.part
+include ../tpch/insert_orders.slt.part
+include ../tpch/insert_part.slt.part
+include ../tpch/insert_partsupp.slt.part
+include ../tpch/insert_supplier.slt.part
+include ../tpch/insert_region.slt.part
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt b/backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt
new file mode 100644
index 0000000000000..801d8574bc267
--- /dev/null
+++ b/backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt
@@ -0,0 +1,22 @@
+include ../tpch/q1.slt.part
+include ../tpch/q2.slt.part
+include ../tpch/q3.slt.part
+include ../tpch/q4.slt.part
+include ../tpch/q5.slt.part
+include ../tpch/q6.slt.part
+include ../tpch/q7.slt.part
+include ../tpch/q8.slt.part
+include ../tpch/q9.slt.part
+include ../tpch/q10.slt.part
+include ../tpch/q11.slt.part
+include ../tpch/q12.slt.part
+include ../tpch/q13.slt.part
+include ../tpch/q14.slt.part
+include ../tpch/q15.slt.part
+include ../tpch/q16.slt.part
+include ../tpch/q17.slt.part
+include ../tpch/q18.slt.part
+include ../tpch/q19.slt.part
+include ../tpch/q20.slt.part
+include ../tpch/q21.slt.part
+include ../tpch/q22.slt.part
\ No newline at end of file
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt b/backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt
new file mode 100644
index 0000000000000..7c7334cc222d3
--- /dev/null
+++ b/backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt
@@ -0,0 +1,50 @@
+include ../tpch/q1.slt.part
+include ../tpch/q2.slt.part
+include ../tpch/q3.slt.part
+include ../tpch/q4.slt.part
+include ../tpch/q5.slt.part
+include ../tpch/q6.slt.part
+include ../tpch/q7.slt.part
+include ../tpch/q8.slt.part
+include ../tpch/q9.slt.part
+include ../tpch/q10.slt.part
+include ../tpch/q11.slt.part
+include ../tpch/q12.slt.part
+include ../tpch/q13.slt.part
+include ../tpch/q14.slt.part
+include ../tpch/q15.slt.part
+include ../tpch/q16.slt.part
+include ../tpch/q17.slt.part
+include ../tpch/q18.slt.part
+include ../tpch/q19.slt.part
+include ../tpch/q20.slt.part
+include ../tpch/q21.slt.part
+include ../tpch/q22.slt.part
+
+# Test deletes and updates should work as per normal.
+include ./delete.slt.part
+include ./insert.slt.part
+include ../tpch/q1.slt.part
+include ../tpch/q2.slt.part
+include ../tpch/q3.slt.part
+include ../tpch/q4.slt.part
+include ../tpch/q5.slt.part
+include ../tpch/q6.slt.part
+include ../tpch/q7.slt.part
+include ../tpch/q8.slt.part
+include ../tpch/q9.slt.part
+include ../tpch/q10.slt.part
+include ../tpch/q11.slt.part
+include ../tpch/q12.slt.part
+include ../tpch/q13.slt.part
+include ../tpch/q14.slt.part
+include ../tpch/q15.slt.part
+include ../tpch/q16.slt.part
+include ../tpch/q17.slt.part
+include ../tpch/q18.slt.part
+include ../tpch/q19.slt.part
+include ../tpch/q20.slt.part
+include ../tpch/q21.slt.part
+include ../tpch/q22.slt.part
+
+include ../tpch/drop_tables.slt.part
\ No newline at end of file
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
index f047cc9a49c84..4d57be1c8231c 100644
--- a/ci/docker-compose.yml
+++ b/ci/docker-compose.yml
@@ -76,6 +76,7 @@ services:
       - mysql
       - db
       - message_queue
+      - schemaregistry
     volumes:
       - ..:/risingwave
 
@@ -128,6 +129,7 @@ services:
       - ELASTIC_PASSWORD=risingwave
     ports:
       - 9200:9200
+
   clickhouse-server:
     image: clickhouse/clickhouse-server:23.3.8.21-alpine
     container_name: clickhouse-server-1
@@ -137,4 +139,46 @@ services:
       - "9000:9000"
       - "9004:9004"
     expose:
-      - 9009
\ No newline at end of file
+      - 9009
+
+# Temporary workaround for json schema registry test since redpanda only supports
+# protobuf/avro schema registry. Should be removed after the support.
+# Related tracking issue:
+# https://github.com/redpanda-data/redpanda/issues/1878
+  zookeeper:
+    container_name: zookeeper
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - "2181:2181"
+
+  schemaregistry:
+    container_name: schemaregistry
+    hostname: schemaregistry
+    image: confluentinc/cp-schema-registry:latest
+    depends_on:
+      - kafka
+    ports:
+      - "8082:8082"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schemaregistry
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+      SCHEMA_REGISTRY_LISTENERS: http://schemaregistry:8082
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
+      SCHEMA_REGISTRY_DEBUG: 'true'
+
+  kafka:
+    container_name: kafka
+    image: confluentinc/cp-kafka:latest
+    ports:
+      - "29093:29093"
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
diff --git a/ci/scripts/backwards-compat-test.sh b/ci/scripts/backwards-compat-test.sh
index cbbc87cb6c08a..1c1b4d388faa9 100755
--- a/ci/scripts/backwards-compat-test.sh
+++ b/ci/scripts/backwards-compat-test.sh
@@ -23,128 +23,18 @@ done
 shift $((OPTIND -1))
 
 # profile is either ci-dev or ci-release
-if [[ "$profile" != "ci-dev" ]] && [[ "$profile" != "ci-release" ]]; then
+if [[ "$profile" == "ci-dev" ]]; then
+  echo "Running in ci-dev mode"
+elif [[ "$profile" == "ci-release" ]]; then
+  echo "Running in ci-release mode"
+else
     echo "Invalid option: profile must be either ci-dev or ci-release" 1>&2
     exit 1
 fi
 
-################################### ENVIRONMENT VARIABLES
+source backwards-compat-tests/scripts/utils.sh
 
-LOG_DIR=.risingwave/log
-mkdir -p "$LOG_DIR"
-
-QUERY_LOG_FILE="$LOG_DIR/query.log"
-
-# TODO(kwannoel): automatically derive this by:
-# 1. Fetching major version.
-# 2. Find the earliest minor version of that major version.
-TAG=v0.18.0
-# Duration to wait for recovery (seconds)
-RECOVERY_DURATION=20
-
-################################### TEST UTILIIES
-
-assert_not_empty() {
-  set +e
-  if [[ $(wc -l < "$1" | sed 's/^ *//g') -gt 1 ]]; then
-    echo "assert_not_empty PASSED for $1"
-  else
-    echo "assert_not_empty FAILED for $1"
-    buildkite-agent artifact upload "$1"
-    exit 1
-  fi
-  set -e
-}
-
-assert_eq() {
-  set +e
-  if [[ -z $(diff "$1" "$2") ]]; then
-    echo "assert_eq PASSED for $1 and $2"
-  else
-    echo "FAILED"
-    buildkite-agent artifact upload "$1"
-    buildkite-agent artifact upload "$2"
-    exit 1
-  fi
-  set -e
-}
-
-################################### QUERIES
-
-run_sql () {
-    psql -h localhost -p 4566 -d dev -U root -c "$@"
-}
-
-seed_table() {
-  START="$1"
-  END="$2"
-  for i in $(seq "$START" "$END")
-  do
-    run_sql "INSERT into t values ($i, $i);" 1>$QUERY_LOG_FILE 2>&1
-  done
-  run_sql "flush;"
-}
-
-random_delete() {
-  START=$1
-  END=$2
-  COUNT=$3
-  for i in $(seq 1 "$COUNT")
-  do
-    run_sql "DELETE FROM t WHERE v1 = $(("$RANDOM" % END));" 1>$QUERY_LOG_FILE 2>&1
-  done
-  run_sql "flush;"
-}
-
-random_update() {
-  START=$1
-  END=$2
-  COUNT=$3
-  for _i in $(seq 1 "$COUNT")
-  do
-    run_sql "UPDATE t SET v2 = v2 + 1 WHERE v1 = $(("$RANDOM" % END));" 1>$QUERY_LOG_FILE 2>&1
-  done
-  run_sql "flush;"
-}
-
-# Setup table and materialized view.
-# Run updates and deletes on the table.
-# Get the results.
-# TODO: Run nexmark, tpch queries
-run_sql_old_cluster() {
-  run_sql "CREATE TABLE t(v1 int primary key, v2 int);"
-
-  seed_table 1 10000
-
-  run_sql "CREATE MATERIALIZED VIEW m as SELECT * from t;" &
-  CREATE_MV_PID=$!
-
-  seed_table 10001 20000
-
-  random_update 1 20000 1000
-
-  random_delete 1 20000 1000
-
-  wait $CREATE_MV_PID
-
-  run_sql "CREATE MATERIALIZED VIEW m2 as SELECT v1, sum(v2) FROM m GROUP BY v1;"
-
-  run_sql "select * from m ORDER BY v1;" > BEFORE_1
-  run_sql "select * from m2 ORDER BY v1;" > BEFORE_2
-}
-
-# Just check if the results are the same as old cluster.
-run_sql_new_cluster() {
-  run_sql "SELECT * from m ORDER BY v1;" > AFTER_1
-  run_sql "select * from m2 ORDER BY v1;" > AFTER_2
-}
-
-run_updates_and_deletes_new_cluster() {
-  random_update 1 20000 1000
-  random_delete 1 20000 1000
-}
-
-################################### CLUSTER CONFIGURATION
+################################### Main
 
 configure_rw() {
 echo "--- Setting up cluster config"
@@ -157,6 +47,8 @@ full-without-monitoring:
     - use: compute-node
     - use: frontend
     - use: compactor
+    - use: zookeeper
+    - use: kafka
 EOF
 
 cat <<EOF > risedev-components.user.env
@@ -164,15 +56,7 @@ RISEDEV_CONFIGURED=true
 
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-# FIXME: Don't use kafka for now,
-# Until 1.0, then we can re-enable it...
-# This is because previous versions of risedev-tool (from previous releases)
-# fetch kafka from clcdn.apache.org which only maintains the latest few
-# versions of kafka.
-# This comment belongs to a PR for the release of 1.0.
-# In this PR, we also change the source of kafka bin to downloads.apache.org,
-# which maintain old versions of kafka (until 2012).
-# ENABLE_KAFKA=true
+ENABLE_KAFKA=true
 
 # Fetch risingwave binary from release.
 ENABLE_BUILD_RUST=false
@@ -180,85 +64,47 @@ ENABLE_BUILD_RUST=false
 # Ensure it will link the all-in-one binary from our release.
 ENABLE_ALL_IN_ONE=true
 
-# ENABLE_RELEASE_PROFILE=true
+# Even if CI is release profile, we won't ever
+# build the binaries from scratch.
+# So we just use target/debug for simplicity.
+ENABLE_RELEASE_PROFILE=false
 EOF
 }
 
-configure_latest_rw() {
-cat <<EOF > risedev-profiles.user.yml
-full-without-monitoring:
-  steps:
-    - use: minio
-    - use: etcd
-    - use: meta-node
-    - use: compute-node
-    - use: frontend
-    - use: compactor
-EOF
+setup_old_cluster() {
+  echo "--- Build risedev for $OLD_VERSION, it may not be backwards compatible"
+  git config --global --add safe.directory /risingwave
+  git checkout "v${OLD_VERSION}-rc"
+  cargo build -p risedev
+  OLD_URL=https://github.com/risingwavelabs/risingwave/releases/download/v${OLD_VERSION}/risingwave-v${OLD_VERSION}-x86_64-unknown-linux.tar.gz
+  wget $OLD_URL
+  tar -xvf risingwave-v${OLD_VERSION}-x86_64-unknown-linux.tar.gz
+  mv risingwave target/debug/risingwave
+
+  echo "--- Start cluster on tag $OLD_VERSION"
+  git config --global --add safe.directory /risingwave
 }
 
-echo "--- Configuring RW"
-configure_rw
-
-echo "--- Build risedev for $TAG, it may not be backwards compatible"
-git config --global --add safe.directory /risingwave
-git checkout "${TAG}-rc"
-cargo build -p risedev
-
-echo "--- Setup old release $TAG"
-wget "https://github.com/risingwavelabs/risingwave/releases/download/${TAG}/risingwave-${TAG}-x86_64-unknown-linux.tar.gz"
-tar -xvf risingwave-${TAG}-x86_64-unknown-linux.tar.gz
-mkdir -p target/debug
-cp risingwave target/debug/risingwave
-
-echo "--- Teardown any old cluster"
-set +e
-./risedev down
-set -e
-
-echo "--- Start cluster on tag $TAG"
-git config --global --add safe.directory /risingwave
-# NOTE(kwannoel): We use this config because kafka encounters errors upon cluster restart,
-# If previous kafka topics and partitions were not removed.
-./risedev d full-without-monitoring && rm .risingwave/log/*
-pushd .risingwave/log/
-buildkite-agent artifact upload "./*.log"
-popd
-
-# TODO(kwannoel): Run nexmark queries + tpch queries.
-# TODO(kwannoel): Refactor this into a rust binary + test files for better maintainability.
-echo "--- Running Queries Old Cluster @ $TAG"
-run_sql_old_cluster
-
-echo "--- Kill cluster on tag $TAG"
-./risedev k
-
-echo "--- Setup Risingwave @ $RW_COMMIT"
-download_and_prepare_rw $profile common
-
-echo "--- Start cluster on latest"
-configure_rw
-./risedev d full-without-monitoring
-
-echo "--- Wait ${RECOVERY_DURATION}s for Recovery on Old Cluster Data"
-sleep $RECOVERY_DURATION
-
-echo "--- Running Queries New Cluster"
-run_sql_new_cluster
-
-echo "--- Sanity Checks"
-echo "AFTER_1"
-cat AFTER_1 | tail -n 100
-echo "AFTER_2"
-cat AFTER_2 | tail -n 100
+setup_new_cluster() {
+  echo "--- Setup Risingwave @ $RW_COMMIT"
+  git checkout -
+  download_and_prepare_rw $profile common
+  # Make sure we always start w/o old config
+  rm -r .risingwave/config
+}
 
-echo "--- Comparing results"
-assert_eq BEFORE_1 AFTER_1
-assert_eq BEFORE_2 AFTER_2
-assert_not_empty BEFORE_1
-assert_not_empty BEFORE_2
-assert_not_empty AFTER_1
-assert_not_empty AFTER_2
+main() {
+  set -euo pipefail
+  # Make sure we have all the branches
+  git fetch --all
+  get_rw_versions
+  setup_old_cluster
+  configure_rw
+  seed_old_cluster "$OLD_VERSION"
+
+  setup_new_cluster
+  configure_rw
+  validate_new_cluster "$NEW_VERSION"
+}
 
-echo "--- Running Updates and Deletes on new cluster should not fail"
-run_updates_and_deletes_new_cluster
\ No newline at end of file
+main
\ No newline at end of file
diff --git a/ci/scripts/check.sh b/ci/scripts/check.sh
index 36ea48aca6b1f..26b03343e4974 100755
--- a/ci/scripts/check.sh
+++ b/ci/scripts/check.sh
@@ -35,7 +35,7 @@ sccache --show-stats
 sccache --zero-stats
 
 echo "--- Build documentation"
-cargo doc --document-private-items --no-deps
+RUSTDOCFLAGS="-Dwarnings -Arustdoc::private_intra_doc_links" cargo doc --document-private-items --no-deps
 
 echo "--- Show sccache stats"
 sccache --show-stats
@@ -50,5 +50,7 @@ sccache --zero-stats
 
 echo "--- Run audit check"
 cargo audit \
-  --ignore RUSTSEC-2023-0052 # https://github.com/risingwavelabs/risingwave/issues/11842
+  --ignore RUSTSEC-2023-0052 --ignore RUSTSEC-2022-0093
+ # https://github.com/risingwavelabs/risingwave/issues/11842
+ # https://github.com/risingwavelabs/risingwave/issues/11986
 
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index 0e9b041438d83..5c710b8607f5b 100644
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -13,6 +13,7 @@ export CARGO_MAKE_PRINT_TIME_SUMMARY=true
 export MINIO_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/minio
 export MCLI_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/mc
 export GCLOUD_DOWNLOAD_TGZ=https://ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
+export NEXTEST_HIDE_PROGRESS_BAR=true
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT
diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
index 847007f574368..def1368641641 100755
--- a/ci/scripts/e2e-source-test.sh
+++ b/ci/scripts/e2e-source-test.sh
@@ -35,6 +35,7 @@ echo "--- Prepare data"
 cp src/connector/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc
 cp src/connector/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc
 cp src/connector/src/test_data/complex-schema ./proto-complex-schema
+cp src/connector/src/test_data/complex-schema.json ./json-complex-schema
 
 
 echo "--- e2e, ci-1cn-1fe, mysql & postgres cdc"
diff --git a/ci/scripts/e2e-test-parallel-in-memory.sh b/ci/scripts/e2e-test-parallel-in-memory.sh
index 1a2d0812766f3..dec93557ffef7 100755
--- a/ci/scripts/e2e-test-parallel-in-memory.sh
+++ b/ci/scripts/e2e-test-parallel-in-memory.sh
@@ -36,7 +36,7 @@ cargo make ci-kill
 echo "--- e2e, ci-3cn-3fe-in-memory, batch"
 cargo make ci-start ci-3cn-3fe-in-memory
 sqllogictest ${host_args} -d dev './e2e_test/ddl/**/*.slt' --junit "parallel-in-memory-batch-ddl-${profile}" --label in-memory
-sqllogictest ${host_args} -d dev './e2e_test/visibility_mode/barrier.slt' -j 16 --junit "parallel-in-memory-batch-${profile}" --label in-memory
+sqllogictest ${host_args} -d dev './e2e_test/batch/**/*.slt' -j 16 --junit "parallel-in-memory-batch-${profile}" --label in-memory
 
 echo "--- Kill cluster"
 cargo make ci-kill
diff --git a/ci/scripts/run-fuzz-test.sh b/ci/scripts/run-fuzz-test.sh
index 8127797e4055a..5823d6f1e848e 100755
--- a/ci/scripts/run-fuzz-test.sh
+++ b/ci/scripts/run-fuzz-test.sh
@@ -8,7 +8,7 @@ export RUST_LOG=info
 
 if [[ $RUN_SQLSMITH_FRONTEND -eq "1" ]]; then
     echo "--- Run sqlsmith frontend tests"
-     NEXTEST_PROFILE=ci cargo nextest run --package risingwave_sqlsmith --features "enable_sqlsmith_unit_test" 2> >(tee);
+    NEXTEST_PROFILE=ci cargo nextest run --package risingwave_sqlsmith --features "enable_sqlsmith_unit_test"
 fi
 
 extract_error_sql() {
diff --git a/ci/scripts/run-unit-test.sh b/ci/scripts/run-unit-test.sh
index f5afdc017e52c..c1b7a1b71782d 100755
--- a/ci/scripts/run-unit-test.sh
+++ b/ci/scripts/run-unit-test.sh
@@ -12,8 +12,8 @@ cd ${REPO_ROOT}
 
 echo "+++ Run unit tests with coverage"
 # use tee to disable progress bar
-NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point 2> >(tee);
+NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point --workspace --exclude risingwave_simulation
 
 echo "--- Codecov upload coverage reports"
 curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov
-./codecov -t "$CODECOV_TOKEN" -s . -F rust
\ No newline at end of file
+./codecov -t "$CODECOV_TOKEN" -s . -F rust
diff --git a/ci/scripts/standalone-utils.sh b/ci/scripts/standalone-utils.sh
index 64bca37a1d7de..4461331c28bfb 100755
--- a/ci/scripts/standalone-utils.sh
+++ b/ci/scripts/standalone-utils.sh
@@ -24,7 +24,7 @@ start_standalone() {
         --listen-addr 127.0.0.1:5688 \
         --prometheus-listener-addr 127.0.0.1:1222 \
         --advertise-addr 127.0.0.1:5688 \
-        --metrics-level 1 \
+        --metrics-level info \
         --async-stack-trace verbose \
         --connector-rpc-endpoint 127.0.0.1:50051 \
         --parallelism 4 \
@@ -36,7 +36,7 @@ start_standalone() {
        --advertise-addr 127.0.0.1:4566 \
        --prometheus-listener-addr 127.0.0.1:2222 \
        --health-check-listener-addr 127.0.0.1:6786 \
-       --metrics-level 1 \
+       --metrics-level info \
        --meta-addr http://127.0.0.1:5690" >"$1" 2>&1
 }
 
diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml
index 3d6293fa7998b..0e4ded02791ec 100644
--- a/ci/workflows/pull-request.yml
+++ b/ci/workflows/pull-request.yml
@@ -272,7 +272,7 @@ steps:
       - docker-compose#v4.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     retry: *auto-retry
 
   - label: "unit test (deterministic simulation)"
@@ -422,7 +422,7 @@ steps:
       - "build"
     plugins:
       - docker-compose#v4.9.0:
-          run: ci-flamegraph-env
+          run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
diff --git a/clippy.toml b/clippy.toml
index 465ccb68ced30..bcc3c789ae35a 100644
--- a/clippy.toml
+++ b/clippy.toml
@@ -8,6 +8,11 @@ disallowed-methods = [
     { path = "num_traits::sign::Signed::is_positive", reason = "This returns true for 0.0 but false for 0." },
     { path = "num_traits::sign::Signed::is_negative", reason = "This returns true for -0.0 but false for 0." },
     { path = "num_traits::sign::Signed::signum", reason = "This returns 1.0 for 0.0 but 0 for 0." },
+    { path = "speedate::DateTime::parse_str", reason = "Please use `parse_str_rfc3339` instead." },
+    { path = "speedate::DateTime::parse_bytes", reason = "Please use `parse_bytes_rfc3339` instead." },
+    { path = "speedate::DateTime::parse_bytes_with_config", reason = "Please use `parse_bytes_rfc3339_with_config` instead." },
+    { path = "speedate::Date::parse_str", reason = "Please use `parse_str_rfc3339` instead." },
+    { path = "speedate::Date::parse_bytes", reason = "Please use `parse_bytes_rfc3339` instead." },
 ]
 disallowed-types = [
     { path = "num_traits::AsPrimitive", reason = "Please use `From` or `TryFrom` with `OrderedFloat` instead." },
diff --git a/docker/README.md b/docker/README.md
index 1cb21a9b26805..5b06140bac9c1 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -37,6 +37,7 @@ To ensure you are using the latest version of RisingWave image,
 docker pull ghcr.io/risingwavelabs/risingwave:latest
 ```
 
+### playground
 To start a RisingWave playground, run
 
 ```
@@ -44,7 +45,8 @@ To start a RisingWave playground, run
 docker run -it --pull=always -p 4566:4566 -p 5691:5691 ghcr.io/risingwavelabs/risingwave:latest playground
 ```
 
-To start a RisingWave cluster, run
+### minio
+To start a RisingWave cluster with minio backend, run
 
 ```
 # Start all components
@@ -53,6 +55,40 @@ docker-compose up
 
 It will start a minio, a meta node, a compute node, a frontend, a compactor, a prometheus and a redpanda instance.
 
+### s3 and other s3-compatible storage backend
+To start a RisingWave cluster with s3 backend, configure the aws credit in [aws.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/aws.env).
+If you want to use some s3 compatible storage like Tencent Cloud COS, just configure one more [endpoint](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/aws.env#L7).
+After configuring the environment and fill in your [bucket name and data directory](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-s3.yml#L196), run
+
+```
+# Start all components
+docker-compose -f docker-compose-with-s3.yml up
+```
+
+It will run with s3 (compatible) object storage with a meta node, a compute node, a frontend, a compactor, a prometheus and a redpanda instance.
+
+### Start with other storage products of  public cloud vendors
+To start a RisingWave cluster with other storage backend, like Google Cloud Storage, Alicloud OSS or Azure Blob Storage, configure the authentication information in [multiple_object_storage.env](https://github.com/risingwavelabs/risingwave/blob/main/docker/multiple_object_storage.env), fill in your [bucket name and data directory](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-gcs.yml#L196).
+and run
+
+```
+# Start all components
+docker-compose -f docker-compose-with-xxx.yml up
+```
+
+It will run RisingWave with corresponding (object) storage products.
+
+### Start with HDFS backend
+To start a RisingWave cluster with HDFS, mount your `HADDOP_HOME` in [compactor node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L28), [compute node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L112) [compute node volumes](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L218), fill in the [cluster_name/namenode and data_path](https://github.com/risingwavelabs/risingwave/blob/a2684461e379ce73f8d730982147439e2379de16/docker/docker-compose-with-hdfs.yml#L202),
+and run
+
+```
+# Start all components
+docker-compose -f docker-compose-with-hdfs.yml up
+```
+
+It will run RisingWave with HDFS.
+
 To clean all data, run:
 
 ```
diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json
index 6e8d945a4ba11..b57021d2cef98 100644
--- a/docker/dashboards/risingwave-dev-dashboard.json
+++ b/docker/dashboards/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":27},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":35},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":44},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":52},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":60},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","query":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":60},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":68},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":68},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows output by each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":76},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_name}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":76},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":84},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":84},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":92},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":92},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":100},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":100},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":108},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":108},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":116},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}->{{executor_identity}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Backpressure","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}->{{upstream_fragment_id}}","metric":"","query":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, actor_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,actor_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, actor_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,actor_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Entries","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the size of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Estimated Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":117},"height":null,"hideTimeOverride":false,"id":61,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":118},"height":null,"hideTimeOverride":false,"id":77,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":119},"height":null,"hideTimeOverride":false,"id":80,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":120},"height":null,"hideTimeOverride":false,"id":83,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":121},"height":null,"hideTimeOverride":false,"id":88,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":122},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":122},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(sstable_preload_io_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"preload iops","metric":"","query":"sum(rate(sstable_preload_io_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) ","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":130},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"File Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":130},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_range_reverse_scan_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"backward scan - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_range_reverse_scan_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":138},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":138},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":146},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.999, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.999, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":146},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":154},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p90 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":154},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.9, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p90 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.9, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":162},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":162},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":170},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":170},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_may_exist_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_may_exist_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_may_exist_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_may_exist_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - MayExist","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":178},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter miss count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Bloom Filter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":178},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":186},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) / (sum(rate(file_cache_latency_count{op='get',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss rate @ {{instance}}","metric":"","query":"(sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) / (sum(rate(file_cache_latency_count{op='get',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":186},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter miss rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom-Filter Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":194},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Request Bloom-Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":194},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p90 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":202},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":202},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":210},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":210},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":218},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":218},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":226},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size (Max)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":226},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block Cache 3. Uploader.This metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":234},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":234},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":242},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":242},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":250},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":250},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":258},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":258},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":266},"height":null,"hideTimeOverride":false,"id":125,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success & Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that’s another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":267},"height":null,"hideTimeOverride":false,"id":150,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":268},"height":null,"hideTimeOverride":false,"id":159,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(data_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(meta_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(data_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(meta_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"data_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"size @ {{instance}}","metric":"","query":"data_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"size @ {{instance}}","metric":"","query":"meta_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compute_refill_data_file_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill data file cache - {{extra}} @ {{instance}}","metric":"","query":"sum(rate(compute_refill_data_file_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":269},"height":null,"hideTimeOverride":false,"id":166,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p999 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.999, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p999 - {{method}}","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":270},"height":null,"hideTimeOverride":false,"id":183,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p999 - {{state}}","metric":"","query":"histogram_quantile(0.999, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":271},"height":null,"hideTimeOverride":false,"id":186,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":272},"height":null,"hideTimeOverride":false,"id":190,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":273},"height":null,"hideTimeOverride":false,"id":193,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":274},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":275},"height":null,"hideTimeOverride":false,"id":202,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":276},"height":null,"hideTimeOverride":false,"id":210,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":277},"height":null,"hideTimeOverride":false,"id":219,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":278},"height":null,"hideTimeOverride":false,"id":226,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":279},"height":null,"hideTimeOverride":false,"id":229,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time since this client instance was created (milli seconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Client Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":231,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Broker Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Topic Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":233,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Partition Level Metrics","transformations":[],"transparent":false,"type":"row"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","query":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows output by each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_name}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":33,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}->{{executor_identity}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Backpressure","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}->{{upstream_fragment_id}}","metric":"","query":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Entries","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the size of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Estimated Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cached_entry_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cached entry count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_over_window_cached_entry_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":63,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":79,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":86,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":92,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":113,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size (Max)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":125,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":151,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":160,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","query":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(data_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_refill_filtered_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache refill - filtered @ {{instance}}","metric":"","query":"sum(rate(data_refill_filtered_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(meta_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - data file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - data file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - data file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - meta cache refill @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - meta cache refill @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - meta cache refill @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":169,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":189,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":196,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":199,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":203,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":216,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":225,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":235,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time since this client instance was created (milli seconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Client Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Broker Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":238,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Topic Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":239,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Partition Level Metrics","transformations":[],"transparent":false,"type":"row"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":240,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/docker/docker-compose-with-hdfs.yml b/docker/docker-compose-with-hdfs.yml
index 0e652220bd848..c76d3b3a7ff1d 100644
--- a/docker/docker-compose-with-hdfs.yml
+++ b/docker/docker-compose-with-hdfs.yml
@@ -12,7 +12,7 @@ services:
       - "--prometheus-listener-addr"
       - "0.0.0.0:1260"
       - "--metrics-level"
-      - "1"
+      - "info"
       - "--meta-address"
       - "http://meta-node-0:5690"
       - "--config-path"
@@ -47,7 +47,7 @@ services:
       - "--prometheus-listener-addr"
       - "0.0.0.0:1222"
       - "--metrics-level"
-      - "1"
+      - "info"
       - "--meta-address"
       - "http://meta-node-0:5690"
       - "--connector-rpc-endpoint"
@@ -109,7 +109,6 @@ services:
     depends_on: []
     volumes:
       - "etcd-0:/etcd-data"
-      - "<HADOOP_HOME>:/opt/hadoop/"
     environment:
       - HADOOP_HOME=/opt/hadoop/
     container_name: etcd-0
@@ -138,7 +137,7 @@ services:
       - "--prometheus-listener-addr"
       - "0.0.0.0:2222"
       - "--metrics-level"
-      - "1"
+      - "info"
     expose:
       - "4566"
     ports:
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 57a495b41bd20..107124d5c27b5 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -12,7 +12,7 @@ services:
       - "--prometheus-listener-addr"
       - "0.0.0.0:1260"
       - "--metrics-level"
-      - "1"
+      - "info"
       - "--meta-address"
       - "http://meta-node-0:5690"
       - "--config-path"
@@ -49,7 +49,7 @@ services:
       - "--prometheus-listener-addr"
       - "0.0.0.0:1222"
       - "--metrics-level"
-      - "1"
+      - "info"
       - "--meta-address"
       - "http://meta-node-0:5690"
       - "--connector-rpc-endpoint"
@@ -140,7 +140,7 @@ services:
       - "--prometheus-listener-addr"
       - "0.0.0.0:2222"
       - "--metrics-level"
-      - "1"
+      - "info"
     expose:
       - "4566"
     ports:
diff --git a/docs/developer-guide.md b/docs/developer-guide.md
index 591f42c44c520..e256d1a5ebd83 100644
--- a/docs/developer-guide.md
+++ b/docs/developer-guide.md
@@ -47,6 +47,7 @@ http://ecotrust-canada.github.io/markdown-toc/
 - [Add new dependencies](#add-new-dependencies)
 - [Submit PRs](#submit-prs)
 - [Profiling](#benchmarking-and-profiling)
+- [Understanding RisingWave Macros](#understanding-risingwave-macros)
 
 ## Read the design docs
 
@@ -62,6 +63,8 @@ You can also read the [crate level documentation](https://risingwavelabs.github.
 - The `docs` folder contains the design docs. If you want to learn about how RisingWave is designed and implemented, check out the design docs here.
 - The `dashboard` folder contains RisingWave dashboard v2.
 
+The [src/README.md](../src/README.md) file contains more details about Design Patterns in RisingWave.
+
 ## Set up the development environment
 
 RiseDev is the development mode of RisingWave. To develop RisingWave, you need to build from the source code and run RiseDev. RiseDev can be built on macOS and Linux. It has the following dependencies:
@@ -269,7 +272,10 @@ The Rust components use `tokio-tracing` to handle both logging and tracing. The
 * Third-party libraries: warn
 * Other libraries: debug
 
-If you need to adjust log levels, change the logging filters in `src/utils/runtime/src/lib.rs`.
+If you need to override the default log levels, launch RisingWave with the environment variable `RUST_LOG` set as described [here](https://docs.rs/tracing-subscriber/0.3/tracing_subscriber/filter/struct.EnvFilter.html).
+
+There're also some logs designated for debugging purposes with target names starting with `events::`.
+For example, by setting `RUST_LOG=events::stream::message::chunk=trace`, all chunk messages will be logged as it passes through the executors in the streaming engine. Search in the codebase to find more of them.
 
 
 ## Test your code changes
@@ -466,6 +472,18 @@ You may use that to reproduce it in your local environment. For example:
 MADSIM_TEST_SEED=4 ./risedev sit-test test_backfill_with_upstream_and_snapshot_read
 ```
 
+### Backwards Compatibility tests
+
+This tests backwards compatibility between the earliest minor version
+and latest minor version of Risingwave (e.g. 1.0.0 vs 1.1.0).
+
+You can run it locally with:
+```bash
+./risedev backwards-compat-test
+```
+
+In CI, you can make sure the PR runs it by adding the label `ci/run-backwards-compat-tests`.
+
 ## Miscellaneous checks
 
 For shell code, please run:
@@ -514,4 +532,4 @@ Instructions about submitting PRs are included in the [contribution guidelines](
 
 - [CPU Profiling Guide](./cpu-profiling.md)
 - [Memory (Heap) Profiling Guide](./memory-profiling.md)
-- [Microbench Guide](./microbenchmarks.md)
\ No newline at end of file
+- [Microbench Guide](./microbenchmarks.md)
diff --git a/docs/rustdoc/index.md b/docs/rustdoc/index.md
index 20b7ae161654d..d82b1899c79a9 100644
--- a/docs/rustdoc/index.md
+++ b/docs/rustdoc/index.md
@@ -12,6 +12,8 @@ Here are the rustdocs of the core crates in RisingWave. You can read them to und
 
 <!-- Not all crates are listed here. For example, binary crates and test crates are not included. -->
 
+There's also a [Dash](https://kapeli.com/dash) docset feed containing all crate docs including dependencies, which might be handy for developing RisingWave: [subscribe here](dash-feed://https%3A%2F%2Frisingwavelabs.github.io%2Frisingwave%2FRisingWave.xml)
+
 ### Core Components
 
 - **Frontend**
diff --git a/e2e_test/batch/basic/func.slt.part b/e2e_test/batch/basic/func.slt.part
index b33125b02d285..31ae11c1226d1 100644
--- a/e2e_test/batch/basic/func.slt.part
+++ b/e2e_test/batch/basic/func.slt.part
@@ -443,3 +443,162 @@ select regexp_matches('abcabc', NULL);
 query T
 select regexp_matches('abcabc', 'a', NULL);
 ----
+
+query T
+select regexp_replace('foobarbaz', 'b..', 'X');
+----
+fooXbaz
+
+query T
+select regexp_replace('foobarbaz', 'b..', 'X', 'g');
+----
+fooXX
+
+query T
+select regexp_replace('foobarbaz', 'b(..)', 'X\1Y', 'g');
+----
+fooXarYXazY
+
+query T
+select regexp_replace('foobarbazabc', 'b(..)', 'X\1Y', 'g');
+----
+fooXarYXazYabc
+
+query T
+select regexp_replace('A PostgreSQL function', 'a|e|i|o|u', 'X', 1, 0, 'i');
+----
+X PXstgrXSQL fXnctXXn
+
+query T
+select regexp_replace('A PostgreSQL function', 'a|e|i|o|u', 'X', 1, 3, 'i');
+----
+A PostgrXSQL function
+
+# Using Parenthesized Subexpression in Replacement
+query T
+select regexp_replace('hello hello', '(hel)(lo)', '\2\1');
+----
+lohel hello
+
+# Literal Backslash in Replacement
+query T
+select regexp_replace('path\to\directory', '\\', '/');
+----
+path/to\directory
+
+# Multiple Flags
+query T
+select regexp_replace('HELLO world', 'a|e|i|o|u', 'X', 'ig');
+----
+HXLLX wXrld
+
+# No Match Found
+query T
+select regexp_replace('apple', 'orange', 'fruit');
+----
+apple
+
+# Using `\&` in Replacement
+query T
+select regexp_replace('abc123', 'abc', 'prefix\&suffix');
+----
+prefixabcsuffix123
+
+query error invalid syntax for `regexp_replace`
+select regexp_replace('foobarbaz', 'b..', 'X', 1, 'g');
+
+query error invalid parameters specified in regexp_replace
+select regexp_replace('foobarbaz', 'b..', 'X', 'g', 1);
+
+# With Unicode
+query T
+select regexp_replace('foo💩barbaz', 'b..', 'X');
+----
+foo💩Xbaz
+
+query T
+select regexp_replace('foo💩bar💩baz', 'b..', 'X');
+----
+foo💩X💩baz
+
+query T
+select regexp_replace('foo💩bar💩baz', 'b..', 'X', 'g');
+----
+foo💩X💩X
+
+query T
+select regexp_replace('foo💩bar💩baz', '💩', 'X');
+----
+fooXbar💩baz
+
+query T
+select regexp_replace('💩💩💩💩💩', '💩', 'X', 'g');
+----
+XXXXX
+
+query T
+select regexp_replace('aAA', 'a|e|i|o|u', 'X', 'ig');
+----
+XXX
+
+query T
+select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic');
+----
+💩💩💩💩💩foo🤔️bar亲爱的😭这是🥵爱情❤️‍🔥
+
+query T
+select regexp_count('ABCABCAXYaxy', 'A.');
+----
+3
+
+query T
+select regexp_count('ABCABCAX🤔️axy', 'A.');
+----
+3
+
+query T
+select regexp_count('ABCABCAXYaxy', 'A.', 1, 'i');
+----
+4
+
+query T
+select regexp_count('ABCABCAbXYaxy', 'AB');
+----
+2
+
+query T
+select regexp_count('ABCABCAbXYaxy', 'AB', 1, 'i');
+----
+3
+
+query T
+select regexp_count('ABCABCAXYaxy', '[a-c]', 1, 'i');
+----
+8
+
+query T
+select regexp_count('123ABC#456', '\d{3}');
+----
+2
+
+query T
+select regexp_count('foobarbaz', 'b..', 3);
+----
+2
+
+query error Expected start to be a Unsigned Int32
+select regexp_count('foobarbaz', 'b..', 'g');
+
+query error Expected start to be a Unsigned Int32
+select regexp_count('foobarbaz', 'b..', 'i');
+
+query error invalid regular expression option: "a"
+select regexp_count('foobarbaz', 'b..', 3, 'a');
+
+query error `regexp_count` does not support global flag option
+select regexp_count('foobar', 'b..', 1, 'g');
+
+query T
+select regexp_count('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', '亲爱的😭', 1, 'i');
+----
+1
\ No newline at end of file
diff --git a/e2e_test/batch/functions/array_concat.slt.part b/e2e_test/batch/functions/array_concat.slt.part
index b2cee32de28f5..7c853262c587d 100644
--- a/e2e_test/batch/functions/array_concat.slt.part
+++ b/e2e_test/batch/functions/array_concat.slt.part
@@ -664,17 +664,17 @@ select array_prepend(1::real, array[1]::real[]);
 {1,1}
 
 query T
-select array['2020-01-02 12:34:56 -11:00'::timestamp with time zone::varchar]::timestamp[] || '2020-01-01 12:34:56'::timestamp::date;
+select array['2020-01-02 12:34:56 -11:00'::timestamp with time zone]::timestamp[] || '2020-01-01 12:34:56'::timestamp::date;
 ----
 {"2020-01-02 23:34:56","2020-01-01 00:00:00"}
 
 query T
-select array_append(array['2020-01-02 12:34:56 -11:00'::timestamp with time zone::varchar]::timestamp[], '2020-01-01 12:34:56'::timestamp::date);
+select array_append(array['2020-01-02 12:34:56 -11:00'::timestamp with time zone]::timestamp[], '2020-01-01 12:34:56'::timestamp::date);
 ----
 {"2020-01-02 23:34:56","2020-01-01 00:00:00"}
 
 query T
-select array_prepend('2020-01-01 12:34:56'::timestamp::date, array['2020-01-02 12:34:56 -11:00'::timestamp with time zone::varchar]::timestamp[]);
+select array_prepend('2020-01-01 12:34:56'::timestamp::date, array['2020-01-02 12:34:56 -11:00'::timestamp with time zone]::timestamp[]);
 ----
 {"2020-01-01 00:00:00","2020-01-02 23:34:56"}
 
diff --git a/e2e_test/batch/functions/array_max.slt.part b/e2e_test/batch/functions/array_max.slt.part
new file mode 100644
index 0000000000000..54a0e8cb45d19
--- /dev/null
+++ b/e2e_test/batch/functions/array_max.slt.part
@@ -0,0 +1,72 @@
+query I
+select array_max(array[1, 2, 3]);
+----
+3
+
+query I
+select array_max(array[2, 3, 5, 2, 4]);
+----
+5
+
+query I
+select array_max(array[114514, 114513]);
+----
+114514
+
+query I
+select array_max(array['a', 'b', 'c', 'a']);
+----
+c
+
+query I
+select array_max(array['e💩a', 'f🤔️b', 'c🥵c', 'd🥳d', 'e💩e']);
+----
+f🤔️b
+
+query I
+select array_max(array['2c😅🤔😅️c2', '114🥵514', '30🤣🥳03', '5🥵💩💩🥵5']);
+----
+5🥵💩💩🥵5
+
+query error invalid digit found in string
+select array_max(array['a', 'b', 'c', 114514]);
+
+query error invalid digit found in string
+select array_max(array[114514, 'a', 'b', 'c']);
+
+# i32::MIN & i32::MIN - 1 & i32::MAX
+query I
+select array_max(array[-2147483648, 2147483647, -2147483649]);
+----
+2147483647
+
+# i64::MIN & i64::MIN - 1 & i64::MAX
+query I
+select array_max(array[-9223372036854775808, 9223372036854775807, -9223372036854775809]);
+----
+9223372036854775807
+
+query I
+select array_max(array['a', '', 'c']);
+----
+c
+
+query I
+select array_max(array[3.14, 1.14, 1.14514]);
+----
+3.14
+
+query I
+select array_max(array[3.1415926, 191.14, 114514, 1313.1414]);
+----
+114514
+
+query I
+select array_max(array[1e-4, 1.14514e5, 1.14514e-5]);
+----
+114514
+
+query I
+select array_max(array[date'2002-10-30', date'2023-09-06', date'2017-06-18']);
+----
+2023-09-06
\ No newline at end of file
diff --git a/e2e_test/batch/functions/array_min.slt.part b/e2e_test/batch/functions/array_min.slt.part
new file mode 100644
index 0000000000000..0a252465c58d7
--- /dev/null
+++ b/e2e_test/batch/functions/array_min.slt.part
@@ -0,0 +1,93 @@
+query I
+select array_min(array[1, 2, 3]);
+----
+1
+
+query I
+select array_min(array[2, 3, 5, 2, 4]);
+----
+2
+
+query I
+select array_min(array[114514, 123456]);
+----
+114514
+
+query I
+select array_min(array['a', 'b', 'c', 'a']);
+----
+a
+
+query I
+select array_min(array['e💩a', 'f🤔️e', 'c🥵c', 'g🥳g', 'e💩e']);
+----
+c🥵c
+
+query I
+select array_min(array['901😅🤔😅️109', '114🥵514', '3🤣🥳3', '5🥵💩💩🥵5']);
+----
+114🥵514
+
+query error invalid digit found in string
+select array_min(array['a', 'b', 'c', 114514]);
+
+query error invalid digit found in string
+select array_min(array[114514, 'a', 'b', 'c']);
+
+# i32::MIN & i32::MIN - 1 & i32::MAX
+query I
+select array_min(array[-2147483648, 2147483647, -2147483649]);
+----
+-2147483649
+
+# i64::MIN & i64::MIN - 1 & i64::MAX
+query I
+select array_min(array[-9223372036854775808, 9223372036854775807, -9223372036854775809]);
+----
+-9223372036854775809
+
+query I
+select array_min(array['a', '', 'c']);
+----
+(empty)
+
+query I
+select array_min(array[3.14, 1.14, 1.14514]);
+----
+1.14
+
+query I
+select array_min(array[3.1415926, 191.14, 114514, 1313.1414]);
+----
+3.1415926
+
+query I
+select array_min(array[1e-4, 1.14514e5, 1.14514e-5]);
+----
+0.0000114514
+
+query I
+select array_min(array[date'2002-10-30', date'2023-09-06', date'2017-06-18']);
+----
+2002-10-30
+
+query I
+select array_min(
+    array[
+        '2002-10-30 00:00:00'::timestamp,
+        '2023-09-06 13:10:00'::timestamp,
+        '2017-06-18 12:00:00'::timestamp
+    ]
+);
+----
+2002-10-30 00:00:00
+
+query I
+select array_min(array['\xDE'::bytea, '\xDF'::bytea, '\xDC'::bytea]);
+----
+\xdc
+
+query I
+select array_min(array[NULL, 'a', 'b']);
+----
+a
\ No newline at end of file
diff --git a/e2e_test/batch/functions/array_transform.slt.part b/e2e_test/batch/functions/array_transform.slt.part
new file mode 100644
index 0000000000000..d07a7e5701429
--- /dev/null
+++ b/e2e_test/batch/functions/array_transform.slt.part
@@ -0,0 +1,56 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+query T
+select array_transform('{1,2,3}'::int[], |x| x * 2);
+----
+{2,4,6}
+
+query T
+select array_transform('{1,2,3}'::int[], |x| (x::double precision+0.5));
+----
+{1.5,2.5,3.5}
+
+query T
+select array_transform('{1,2,3}'::int[], |x| (x::double precision+0.5));
+----
+{1.5,2.5,3.5}
+
+query T
+select array_transform(
+    array_transform(
+        array_transform('{1,2,3}'::int[], |x| x * 2),
+        |x| x + 0.5
+    ),
+    |x| concat(x::varchar, '!')
+)
+----
+{2.5!,4.5!,6.5!}
+
+query T
+select array_transform(
+    ARRAY['Apple', 'Airbnb', 'Amazon', 'Facebook', 'Google', 'Microsoft', 'Netflix', 'Uber'],
+    |x| case when x ilike 'A%' then 'A' else 'Other' end
+)
+----
+{A,A,A,Other,Other,Other,Other,Other}
+
+statement ok
+create table t(v int, arr int[]);
+
+statement ok
+insert into t values (4, '{1,2,3}'), (5, '{4,5,6,8}');
+
+# this makes sure `x + 1` is not extracted as common sub-expression by accident. See #11766
+query TT
+select array_transform(arr, |x| x + 1), array_transform(arr, |x| x + 1 + 2) from t;
+----
+{2,3,4} {4,5,6}
+{5,6,7,9} {7,8,9,11}
+
+# this clarifies that we do not support referencing columns.
+statement error
+select array_transform(arr, |x| x + v) from t;
+
+statement ok
+drop table t;
diff --git a/e2e_test/batch/types/timestamptz_utc.slt.part b/e2e_test/batch/types/timestamptz_utc.slt.part
index a75529b30b1cb..5a0fe6ebd13cc 100644
--- a/e2e_test/batch/types/timestamptz_utc.slt.part
+++ b/e2e_test/batch/types/timestamptz_utc.slt.part
@@ -42,6 +42,19 @@ select '2022-10-01T12:00:00Z'::timestamp with time zone;
 ----
 2022-10-01 12:00:00+00:00
 
+query T
+select '2023-11-05 01:40-07:00'::timestamptz;
+----
+2023-11-05 08:40:00+00:00
+
+query T
+select '2023-11-05 01:40-08:00'::timestamptz;
+----
+2023-11-05 09:40:00+00:00
+
+statement error
+select '0'::timestamptz;
+
 query T
 select '2022-10-01 12:00:00+01:00'::timestamp with time zone BETWEEN '2022-10-01T10:59:59Z' AND '2022-10-01T11:00:01Z';
 ----
diff --git a/e2e_test/schema_registry/pb.slt b/e2e_test/schema_registry/pb.slt
index a48f7f77cbd86..9914d3fa04b2f 100644
--- a/e2e_test/schema_registry/pb.slt
+++ b/e2e_test/schema_registry/pb.slt
@@ -14,6 +14,19 @@ FORMAT plain ENCODE protobuf(
         message = 'test.User'
     );
 
+# for multiple schema registry nodes
+statement ok
+create table sr_pb_test_bk with (
+    connector = 'kafka',
+    topic = 'sr_pb_test',
+    properties.bootstrap.server = 'message_queue:29092',
+    scan.startup.mode = 'earliest',
+    message = 'test.User')
+FORMAT plain ENCODE protobuf(
+        schema.registry = 'http://message_queue:8081,http://message_queue:8081',
+        message = 'test.User'
+    );
+
 # Wait for source
 sleep 10s
 
@@ -33,4 +46,7 @@ select min(id), max(id) from sr_pb_test;
 
 
 statement ok
-drop table sr_pb_test;
\ No newline at end of file
+drop table sr_pb_test;
+
+statement ok
+drop table sr_pb_test_bk;
\ No newline at end of file
diff --git a/e2e_test/sink/kafka/debezium1.result b/e2e_test/sink/kafka/debezium1.result
index d65c39be72a38..6c728e55346d9 100644
--- a/e2e_test/sink/kafka/debezium1.result
+++ b/e2e_test/sink/kafka/debezium1.result
@@ -1,10 +1,10 @@
-{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683859888534},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
diff --git a/e2e_test/sink/kafka/debezium2.result b/e2e_test/sink/kafka/debezium2.result
index 47dd9fb96fdad..3012648769070 100644
--- a/e2e_test/sink/kafka/debezium2.result
+++ b/e2e_test/sink/kafka/debezium2.result
@@ -1,11 +1,11 @@
-{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683857916842},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820372},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1683856820370},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549594077},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
diff --git a/e2e_test/sink/kafka/debezium3.result b/e2e_test/sink/kafka/debezium3.result
index a6157e518f0b4..de658fe7157f2 100644
--- a/e2e_test/sink/kafka/debezium3.result
+++ b/e2e_test/sink/kafka/debezium3.result
@@ -1,13 +1,13 @@
-{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	null
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":null,"before":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"op":"d","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381221481},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381191421},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176934},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"RisingWave","table":"RisingWave"},"ts_ms":1684381176930},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.RisingWave.RisingWave.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.RisingWave.RisingWave.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	null
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":null,"before":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"op":"d","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549624995},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549594077},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
+{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka"},"ts_ms":1693549473650},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
diff --git a/e2e_test/sink/remote/jdbc.check.pg.slt b/e2e_test/sink/remote/jdbc.check.pg.slt
index 43247e226e583..82d87571b35ac 100644
--- a/e2e_test/sink/remote/jdbc.check.pg.slt
+++ b/e2e_test/sink/remote/jdbc.check.pg.slt
@@ -33,3 +33,9 @@ select * from t_append_only order by v1, v2;
 1 aaa
 1 bbb
 2 ccc
+
+query I
+select * from biz.t2 order by "aBc";
+----
+1
+2
diff --git a/e2e_test/sink/remote/jdbc.load.slt b/e2e_test/sink/remote/jdbc.load.slt
index 471c0d04d07a0..235348f5bd82c 100644
--- a/e2e_test/sink/remote/jdbc.load.slt
+++ b/e2e_test/sink/remote/jdbc.load.slt
@@ -129,6 +129,26 @@ CREATE SINK s_pg_0 FROM tt1 WITH (
     force_append_only=true
 );
 
+statement ok
+CREATE TABLE tt2 (
+    "aBc" INTEGER PRIMARY KEY
+);
+
+statement ok
+CREATE SINK s_pg_t2 FROM tt2 WITH (
+    connector='jdbc',
+    jdbc.url='jdbc:postgresql://db:5432/test?user=test&password=connector',
+    table.name='t2',
+    schema.name='biz',
+    type='append-only',
+    force_append_only=true
+);
+
+statement ok
+INSERT INTO tt2 VALUES
+    (1),
+    (2);
+
 statement ok
 INSERT INTO tt1 VALUES
     (1, 'aaa'),
diff --git a/e2e_test/sink/remote/pg_create_table.sql b/e2e_test/sink/remote/pg_create_table.sql
index c91792bb5c5db..c899b788e3506 100644
--- a/e2e_test/sink/remote/pg_create_table.sql
+++ b/e2e_test/sink/remote/pg_create_table.sql
@@ -73,3 +73,7 @@ CREATE TABLE biz.t_types (
     array_column VARCHAR[],
     array_column2 DECIMAL[]
 );
+
+CREATE TABLE biz.t2 (
+    "aBc" INTEGER PRIMARY KEY
+);
diff --git a/e2e_test/source/basic/kafka.slt b/e2e_test/source/basic/kafka.slt
index 1d5801cfe4d23..782b4cca7a7f9 100644
--- a/e2e_test/source/basic/kafka.slt
+++ b/e2e_test/source/basic/kafka.slt
@@ -355,6 +355,14 @@ create source s26 (id int) with (
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE BYTES
 
+statement ok
+create table s27 with (
+  connector = 'kafka',
+  topic = 'json_schema_without_schema',
+  properties.bootstrap.server = 'message_queue:29092',
+  scan.startup.mode = 'earliest'
+) FORMAT PLAIN ENCODE JSON (schema.location = 'file:///risingwave/json-complex-schema')
+
 statement ok
 CREATE TABLE mongo_customers (
 	_id BIGINT PRIMARY KEY,
@@ -754,6 +762,11 @@ select count(*) from source_with_rdkafka_props
 ----
 4
 
+query TFITT
+select * from s27
+----
+(9.5,7,12) 12.5 1 An ice sculpture {cold,ice}
+
 statement ok
 drop materialized view source_mv1
 
@@ -814,6 +827,9 @@ drop source s23
 statement ok
 drop source s24
 
+statement ok
+drop table s27
+
 statement ok
 DROP TABLE mongo_customers;
 
diff --git a/e2e_test/source/basic/kinesis.slt b/e2e_test/source/basic/kinesis.slt
new file mode 100644
index 0000000000000..307e40bd0d73a
--- /dev/null
+++ b/e2e_test/source/basic/kinesis.slt
@@ -0,0 +1,11 @@
+# reject kinesis source if the start up mode is not sequence number but sequence number is provided
+statement error
+create source s ( a int, b varchar )
+with (
+  connector = 'kinesis',
+  stream = 'my_stream',
+  scan.startup.sequence_number = '4950',
+  aws.region = 'us-east-1',
+  kinesis.credentials.access = 'my_access_key_id',
+  kinesis.credentials.secret = 'my_secret_access_key',
+) format plain encode json;
diff --git a/e2e_test/source/basic/nosim_kafka.slt b/e2e_test/source/basic/nosim_kafka.slt
index bc398be748625..945f60e732fb8 100644
--- a/e2e_test/source/basic/nosim_kafka.slt
+++ b/e2e_test/source/basic/nosim_kafka.slt
@@ -68,6 +68,23 @@ CREATE TABLE debezium_compact (PRIMARY KEY(order_id)) with (
     kafka.scan.startup.mode = 'earliest'
 ) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081');
 
+statement ok
+CREATE TABLE kafka_json_schema_plain with (
+    connector = 'kafka',
+    kafka.topic = 'kafka_json_schema',
+    kafka.brokers = 'kafka:9093',
+    kafka.scan.startup.mode = 'earliest'
+) FORMAT PLAIN ENCODE JSON (schema.registry = 'http://schemaregistry:8082');
+
+statement ok
+CREATE TABLE kafka_json_schema_upsert (PRIMARY KEY(id))
+with (
+    connector = 'kafka',
+    kafka.topic = 'kafka_upsert_json_schema',
+    kafka.brokers = 'kafka:9093',
+    kafka.scan.startup.mode = 'earliest'
+) FORMAT UPSERT ENCODE JSON (schema.registry = 'http://schemaregistry:8082');
+
 statement ok
 flush;
 
@@ -141,6 +158,17 @@ select count(*) from debezium_compact;
 ----
 2
 
+query TFITT
+select * from kafka_json_schema_plain
+----
+(9.5,7,12) 12.5 1 An ice sculpture {cold,ice}
+
+query TFITT
+select * from kafka_json_schema_upsert order by id
+----
+(9.5,7,12) 1 23 An ice sculpture {cold,ice}
+(9.5,7,12) 2 12.5 An ice sculpture {cold,ice}
+
 statement ok
 DROP TABLE upsert_avro_json_default_key;
 
@@ -158,4 +186,10 @@ statement ok
 DROP TABLE debezium_non_compact;
 
 statement ok
-DROP TABLE debezium_compact;
\ No newline at end of file
+DROP TABLE debezium_compact;
+
+statement ok
+DROP TABLE kafka_json_schema_plain;
+
+statement ok
+DROP TABLE kafka_json_schema_upsert;
\ No newline at end of file
diff --git a/e2e_test/source/cdc/cdc.check.slt b/e2e_test/source/cdc/cdc.check.slt
index 876d41d9adcbd..f9feed9429d2c 100644
--- a/e2e_test/source/cdc/cdc.check.slt
+++ b/e2e_test/source/cdc/cdc.check.slt
@@ -46,3 +46,8 @@ query I
 select count(*) from person_rw;
 ----
 3
+
+query I
+select count(*) from tt3_rw;
+----
+2
diff --git a/e2e_test/source/cdc/cdc.load.slt b/e2e_test/source/cdc/cdc.load.slt
index bd2472f9947a4..4cfacf120648e 100644
--- a/e2e_test/source/cdc/cdc.load.slt
+++ b/e2e_test/source/cdc/cdc.load.slt
@@ -1,5 +1,9 @@
 # CDC source basic test
 
+# enable cdc backfill in ci
+statement ok
+set cdc_backfill='true';
+
 statement ok
 create table products ( id INT,
  name STRING,
@@ -106,6 +110,38 @@ create table orders_2 (
  server.id = '5088'
 );
 
+statement error
+create table tt3_rw (
+   v1 int,
+   v2 timestamp,
+   PRIMARY KEY (v1)
+) with (
+ connector = 'mysql-cdc',
+ hostname = 'mysql',
+ port = '3306',
+ username = 'root',
+ password = '123456',
+ database.name = 'my@db',
+ table.name = 'tt3',
+ server.id = '5089'
+);
+
+statement ok
+create table tt3_rw (
+   v1 int,
+   v2 timestamptz,
+   PRIMARY KEY (v1)
+) with (
+ connector = 'mysql-cdc',
+ hostname = 'mysql',
+ port = '3306',
+ username = 'root',
+ password = '123456',
+ database.name = 'my@db',
+ table.name = 'tt3',
+ server.id = '5089'
+);
+
 # Some columns missing and reordered (postgres-cdc)
 statement ok
 create table shipments_2 (
diff --git a/e2e_test/source/cdc/mysql_cdc.sql b/e2e_test/source/cdc/mysql_cdc.sql
index 0d8ec2e1020cb..89e5274ac3ee1 100644
--- a/e2e_test/source/cdc/mysql_cdc.sql
+++ b/e2e_test/source/cdc/mysql_cdc.sql
@@ -51,4 +51,7 @@ VALUES (1,1,'no'),
 
 CREATE USER 'dbz'@'%' IDENTIFIED BY '123456';
 GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'dbz'@'%';
-GRANT LOCK TABLES ON `my@db`.* TO 'dbz'@'%';
+
+CREATE TABLE tt3 (v1 int primary key, v2 timestamp);
+INSERT INTO tt3 VALUES (1, '2020-07-30 10:08:22');
+INSERT INTO tt3 VALUES (2, '2020-07-31 10:09:22');
diff --git a/e2e_test/streaming/aggregate/hdr_approx_percentile.slt b/e2e_test/streaming/aggregate/hdr_approx_percentile.slt
new file mode 100755
index 0000000000000..b0969139e92ec
--- /dev/null
+++ b/e2e_test/streaming/aggregate/hdr_approx_percentile.slt
@@ -0,0 +1,289 @@
+# --------
+# Overview
+# --------
+#
+# This is a demo test to show how to use hdr_histogram (http://hdrhistogram.org/)
+# to compute `approx_percentile` in the RisingWave stream engine.
+# It supports non-append-only streams, i.e. streams with updates and deletes are supported.
+#
+# The way it works is that hdr_histogram stores N significant digits of each value,
+# thereby reducing the number of buckets needed to store the distribution.
+#
+# ----------------------
+# Implementation details
+# ----------------------
+#
+# First we create the histogram itself, where each value is encoded as a triple:
+# 1. The sign of the value (1 or -1)
+# 2. The exponent of the value (the power of 10, truncated to an integer)
+# 3. The significand of the value (the digits after the decimal point, truncated to $PRECISION digits)
+#
+# With the following parameters:
+# precision=2 (significand is 2 digits)
+# value=123456
+#
+# The exponent will be 5 (log10(123456) = 5.0915... ~ 5)
+# The significand will be 23 (123456 / 10^5 - 1 = 0.23... ).
+# Then the histogram will store the triple (1, 5, 23)
+#
+# NOTE(kwannoel): We don't have to use base-10 for the exponent and significand.
+# But RisingWave currently only supports log10, ln.
+# So we use base-10 for convenience, and because it's more intuitive for the examples here.
+#
+# The bucket is computed as:
+# (sign*(1.0+mantissa/pow(10.0, 2))*pow(10.0,exponent))::int
+# (1 * (1.0 + 23 / 10^2) * 10^5) = 123000
+#
+# Next, we use window sum aggregation to compute the cumulative frequency of each bucket.
+# | Bucket       | 4 | 311 | 400 | 521 |
+# | Counts       | 3 | 6   | 7   | 8   |
+# | C. Frequency | 3 | 9   | 16  | 24  |
+#
+# Finally we can compute the approximate percentile
+# 1. Compute the sum of all frequencies. (24)
+# 2. Select a percentile (90%)
+# 3. Compute the frequency corresponding to the percentile (24 * 0.9 = 21.6)
+# 4. Find the first bucket with the cumulative frequency >= 21.6 (521)
+#
+# Step 3 and 4 are required for us to use dynamic filter,
+# since we don't support complex expressions in the dynamic filter.
+# In the future when our optimizer can simplify the dynamic filter, we can avoid it.
+#
+# ----------
+# Test Notes
+# ----------
+#
+# We will test the following parameters:
+# - 90 percentile.
+# - 50 percentile.
+# - Deletes.
+# - Compare with percentile_disc. The significand should match.
+
+statement ok
+CREATE TABLE input (value BIGINT);
+
+statement ok
+INSERT INTO input SELECT * FROM generate_series(1,10);
+
+statement ok
+INSERT INTO input SELECT * FROM generate_series(1000, 1100);
+
+statement ok
+INSERT INTO input SELECT * FROM generate_series(10000, 11000);
+
+statement ok
+INSERT INTO input SELECT * FROM generate_series(100000, 110000);
+
+statement ok
+flush;
+
+# Here `pow(10.0, 4)` is responsible for setting the precision of the histogram.
+# "2" is the precision.
+statement ok
+CREATE MATERIALIZED VIEW hdr_histogram AS
+SELECT
+  CASE WHEN value<0 THEN -1 ELSE 1 END AS sign,
+  trunc(log10(value))::int AS exponent,
+  trunc(
+    pow(10.0, 4)
+     * (value / pow(10.0, trunc(log10(value))::int) - 1.0))::int AS mantissa,
+  count(*) AS frequency,
+  1 as dummy
+FROM input
+GROUP BY sign, exponent, mantissa;
+
+statement ok
+CREATE MATERIALIZED VIEW hdr_sum AS
+SELECT
+  sum(frequency) AS sum_frequency
+FROM hdr_histogram;
+
+# Here `pow(10.0, 4)` is responsible for recovering the position of the mantissa, with the precision.
+# "2" is the precision.
+statement ok
+CREATE MATERIALIZED VIEW hdr_distribution AS
+SELECT
+  (sign*(1.0+mantissa/pow(10.0, 4))*pow(10.0,exponent))::int AS bucket,
+  (sum(frequency)
+    OVER (PARTITION BY NULL
+          ORDER BY (sign, exponent, mantissa)
+          ROWS UNBOUNDED PRECEDING)
+          ) AS cumulative_frequency
+FROM hdr_histogram g
+GROUP BY sign, exponent, mantissa, frequency
+ORDER BY cumulative_frequency;
+
+statement ok
+CREATE INDEX hdr_distribution_idx ON hdr_distribution (cumulative_frequency);
+
+########## 90-percentile ##########
+
+statement ok
+CREATE MATERIALIZED VIEW frequency_at_90 AS
+SELECT sum_frequency * 0.9 AS scaled_sum_freq FROM hdr_sum;
+
+statement ok
+CREATE MATERIALIZED VIEW approx_percentile_90_percent AS
+SELECT bucket AS approximate_percentile
+FROM hdr_distribution x, frequency_at_90 y
+WHERE x.cumulative_frequency >= y.scaled_sum_freq
+ORDER BY cumulative_frequency
+LIMIT 1;
+
+query I
+SELECT * FROM approx_percentile_90_percent;
+----
+108880
+
+query I
+SELECT percentile_disc(0.9) WITHIN GROUP (ORDER BY value) FROM input;
+----
+108889
+
+query B
+SELECT
+ (abs(percentile_disc - approximate_percentile) / percentile_disc) <= 0.0001
+FROM approx_percentile_90_percent, (SELECT percentile_disc(0.9) WITHIN GROUP (ORDER BY value) FROM input);
+----
+t
+
+statement ok
+DROP MATERIALIZED VIEW approx_percentile_90_percent;
+
+statement ok
+DROP MATERIALIZED VIEW frequency_at_90;
+
+########## 50-percentile ##########
+
+statement ok
+CREATE MATERIALIZED VIEW frequency_at_50 AS
+SELECT sum_frequency * 0.5 AS scaled_sum_freq FROM hdr_sum;
+
+statement ok
+CREATE MATERIALIZED VIEW approx_percentile_50_percent AS
+SELECT bucket AS approximate_percentile
+FROM hdr_distribution x, frequency_at_50 y
+WHERE x.cumulative_frequency >= y.scaled_sum_freq
+ORDER BY cumulative_frequency
+LIMIT 1;
+
+query I
+SELECT * FROM approx_percentile_50_percent;
+----
+104440
+
+query I
+SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input;
+----
+104444
+
+query B
+SELECT
+ (abs(percentile_disc - approximate_percentile) / percentile_disc) <= 0.0001
+FROM approx_percentile_50_percent, (SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input);
+----
+t
+
+### Delete > 100000
+
+statement ok
+DELETE FROM input WHERE value > 100000;
+
+statement ok
+flush;
+
+query I
+SELECT * FROM approx_percentile_50_percent;
+----
+10445
+
+query I
+SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input;
+----
+10445
+
+query B
+SELECT
+ (abs(percentile_disc - approximate_percentile) / percentile_disc) <= 0.0001
+FROM approx_percentile_50_percent, (SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input);
+----
+t
+
+### Delete > 10000
+
+statement ok
+DELETE FROM input WHERE value > 10000;
+
+statement ok
+flush;
+
+query I
+SELECT * FROM approx_percentile_50_percent;
+----
+1045
+
+query I
+SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input;
+----
+1045
+
+query B
+SELECT
+ (abs(percentile_disc - approximate_percentile) / percentile_disc) <= 0.0001
+FROM approx_percentile_50_percent, (SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input);
+----
+t
+
+
+### Delete > 100
+
+statement ok
+DELETE FROM input WHERE value > 100;
+
+statement ok
+flush;
+
+query I
+SELECT * FROM approx_percentile_50_percent;
+----
+5
+
+query I
+SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input;
+----
+5
+
+query B
+SELECT
+ percentile_disc = (approximate_percentile::int)
+FROM approx_percentile_50_percent, (SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY value) FROM input);
+----
+t
+
+### Cleanup 50-percentile
+
+statement ok
+DROP MATERIALIZED VIEW approx_percentile_50_percent;
+
+statement ok
+DROP MATERIALIZED VIEW frequency_at_50;
+
+########## Cleanup ##########
+
+statement ok
+DELETE FROM input WHERE value < 1400;
+
+statement ok
+DROP INDEX hdr_distribution_idx;
+
+statement ok
+DROP MATERIALIZED VIEW hdr_distribution;
+
+statement ok
+DROP MATERIALIZED VIEW hdr_sum;
+
+statement ok
+DROP MATERIALIZED VIEW hdr_histogram;
+
+statement ok
+DROP TABLE input;
\ No newline at end of file
diff --git a/e2e_test/streaming/rate_limit.slt b/e2e_test/streaming/rate_limit.slt
new file mode 100644
index 0000000000000..95e87b9fd681c
--- /dev/null
+++ b/e2e_test/streaming/rate_limit.slt
@@ -0,0 +1,26 @@
+statement ok
+CREATE TABLE t1(v1 int, v2 int);
+
+statement ok
+SET RW_STREAMING_RATE_LIMIT TO 10000;
+
+statement ok
+CREATE MATERIALIZED VIEW m AS SELECT * FROM t1;
+
+# Should take around 5s due to rate limit.
+statement ok
+INSERT INTO t1 select * from generate_series(1,50000);
+
+statement ok
+FLUSH;
+
+query I
+select count(*) from m;
+----
+50000
+
+statement ok
+DROP MATERIALIZED VIEW m;
+
+statement ok
+DROP TABLE t1;
\ No newline at end of file
diff --git a/e2e_test/streaming/watermark.slt b/e2e_test/streaming/watermark.slt
index d177d258b7c0c..f145a3a46e282 100644
--- a/e2e_test/streaming/watermark.slt
+++ b/e2e_test/streaming/watermark.slt
@@ -9,10 +9,13 @@ create table t (
 ) append only;
 
 statement ok
-create materialized view mv as select * from t emit on window close;
+create materialized view mv1 as select * from t emit on window close;
 
 statement ok
-insert into t values ('2023-05-06 16:51:00', 1);
+create materialized view mv2 as select t.ts, unnest(Array[1,2,3]) from t emit on window close;
+
+statement ok
+insert into t values ('2023-05-06 16:51:00', 1), ('2023-05-06 16:51:00', 2), ('2023-05-06 16:51:00', 3);
 
 statement ok
 insert into t values ('2023-05-06 16:56:01', 1);
@@ -22,12 +25,31 @@ sleep 5s
 
 skipif in-memory
 query TI
-select * from mv;
+select * from mv1;
 ----
 2023-05-06 16:51:00 1
+2023-05-06 16:51:00 2
+2023-05-06 16:51:00 3
+
+skipif in-memory
+query TI
+select * from mv2;
+----
+2023-05-06 16:51:00 1
+2023-05-06 16:51:00 2
+2023-05-06 16:51:00 3
+2023-05-06 16:51:00 1
+2023-05-06 16:51:00 2
+2023-05-06 16:51:00 3
+2023-05-06 16:51:00 1
+2023-05-06 16:51:00 2
+2023-05-06 16:51:00 3
+
+statement ok
+drop materialized view mv1;
 
 statement ok
-drop materialized view mv;
+drop materialized view mv2;
 
 statement ok
 drop table t;
diff --git a/grafana/common.py b/grafana/common.py
index 43d5fe0ccb016..6f6de73e57058 100644
--- a/grafana/common.py
+++ b/grafana/common.py
@@ -56,6 +56,7 @@
         ]
     )
 
+
 class Layout:
 
     def __init__(self):
@@ -96,6 +97,7 @@ def next_one_third_width_graph(self):
 
 
 class Panels:
+    # Common options for timeseries panels
     common_options = {
         "fillOpacity": 10,
         "interval": "1s",
@@ -197,10 +199,10 @@ def timeseries_latency(self,
         )
 
     def timeseries_latency_ms(self,
-                           title,
-                           description,
-                           targets,
-                           legendCols=["mean"]):
+                              title,
+                              description,
+                              targets,
+                              legendCols=["mean"]):
         gridPos = self.layout.next_half_width_graph()
         return TimeSeries(
             title=title,
@@ -524,7 +526,8 @@ def metric(name, filter=None, node_filter_enabled=True, table_id_filter_enabled=
     if risingwave_name_filter_enabled:
         filters.append("risingwave_name=~\"$instance\"")
     if table_id_filter_enabled:
-        filters.append("table_id=~\"$table\"")
+        # We use "%table|" instead of "%table" here to match empty string for the table_id filter
+        filters.append("table_id=~\"$table|\"")
     if node_filter_enabled:
         filters.append("job=~\"$job\"")
         filters.append("instance=~\"$node\"")
@@ -533,6 +536,7 @@ def metric(name, filter=None, node_filter_enabled=True, table_id_filter_enabled=
     else:
         return name
 
+
 def table_metric(name, filter=None):
     return metric(name, filter, True, True)
 
diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py
index ab9adb7332bfa..6fa036b257f09 100644
--- a/grafana/risingwave-dev-dashboard.dashboard.py
+++ b/grafana/risingwave-dev-dashboard.dashboard.py
@@ -19,114 +19,128 @@
 logging.basicConfig(level=logging.WARN)
 
 
-def section_actor_info(panels):
+def section_actor_info(outer_panels):
+    panels = outer_panels.sub_panel()
     excluded_cols = ['Time', 'Value', '__name__', 'job', 'instance']
     return [
-        panels.row("Actor/Table Id Info"),
-        panels.table_info("Actor Id Info",
-                          "Mapping from actor id to fragment id",
-                          [panels.table_target(f"{metric('actor_info')}")], excluded_cols),
-        panels.table_info("Materialized View Info",
-                          "Mapping from materialized view table id to it's internal table ids",
-                           [panels.table_target(f"{metric('table_info')}")], excluded_cols),
+        outer_panels.row_collapsed(
+            "Actor/Table Id Info",
+        [
+            panels.table_info("Actor Id Info",
+                            "Mapping from actor id to fragment id",
+                            [panels.table_target(f"{metric('actor_info')}")], excluded_cols),
+            panels.table_info("Materialized View Info",
+                            "Mapping from materialized view table id to it's internal table ids",
+                            [panels.table_target(f"{metric('table_info')}")], excluded_cols),
+        ])
     ]
 
 
-def section_cluster_node(panels):
+def section_cluster_node(outer_panels):
+    panels = outer_panels.sub_panel()
     return [
-        panels.row("Cluster Node"),
-        panels.timeseries_count(
-            "Node Count",
-            "The number of each type of RisingWave components alive.",
-            [
-                panels.target(f"sum({metric('worker_num')}) by (worker_type)",
-                              "{{worker_type}}")
-            ],
-            ["last"],
-        ),
-        panels.timeseries_memory(
-            "Node Memory",
-            "The memory usage of each RisingWave component.",
-            [
-                panels.target(
-                    f"avg({metric('process_resident_memory_bytes')}) by (job,instance)",
-                    "{{job}} @ {{instance}}",
-                )
-            ],
-        ),
-        panels.timeseries_cpu(
-            "Node CPU",
-            "The CPU usage of each RisingWave component.",
-            [
-                panels.target(
-                    f"sum(rate({metric('process_cpu_seconds_total')}[$__rate_interval])) by (job,instance)",
-                    "cpu usage (total) - {{job}} @ {{instance}}",
-                ),
+        outer_panels.row_collapsed(
+            "Cluster Node",
+        [
+            panels.timeseries_count(
+                "Node Count",
+                "The number of each type of RisingWave components alive.",
+                [
+                    panels.target(f"sum({metric('worker_num')}) by (worker_type)",
+                                "{{worker_type}}")
+                ],
+                ["last"],
+            ),
+            panels.timeseries_memory(
+                "Node Memory",
+                "The memory usage of each RisingWave component.",
+                [
+                    panels.target(
+                        f"avg({metric('process_resident_memory_bytes')}) by (job,instance)",
+                        "{{job}} @ {{instance}}",
+                    )
+                ],
+            ),
+            panels.timeseries_cpu(
+                "Node CPU",
+                "The CPU usage of each RisingWave component.",
+                [
+                    panels.target(
+                        f"sum(rate({metric('process_cpu_seconds_total')}[$__rate_interval])) by (job,instance)",
+                        "cpu usage (total) - {{job}} @ {{instance}}",
+                    ),
 
-                panels.target(
-                    f"sum(rate({metric('process_cpu_seconds_total')}[$__rate_interval])) by (job,instance) / avg({metric('process_cpu_core_num')}) by (job,instance)",
-                    "cpu usage (avg per core) - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
+                    panels.target(
+                        f"sum(rate({metric('process_cpu_seconds_total')}[$__rate_interval])) by (job,instance) / avg({metric('process_cpu_core_num')}) by (job,instance)",
+                        "cpu usage (avg per core) - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Meta Cluster",
-            "RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the "
-            "rest are the followers.",
-            [
-                panels.target(f"sum({metric('meta_num')}) by (worker_addr,role)",
-                              "{{worker_addr}} @ {{role}}")
-            ],
-            ["last"],
-        ),
+            panels.timeseries_count(
+                "Meta Cluster",
+                "RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the "
+                "rest are the followers.",
+                [
+                    panels.target(f"sum({metric('meta_num')}) by (worker_addr,role)",
+                                "{{worker_addr}} @ {{role}}")
+                ],
+                ["last"],
+            ),
+        ])
     ]
 
 
-def section_recovery_node(panels):
+def section_recovery_node(outer_panels):
+    panels = outer_panels.sub_panel()
     return [
-        panels.row("Recovery"),
-        panels.timeseries_ops(
-            "Recovery Successful Rate",
-            "The rate of successful recovery attempts",
-            [
-                panels.target(f"sum(rate({metric('recovery_latency_count')}[$__rate_interval])) by (instance)",
-                              "{{instance}}")
-            ],
-            ["last"],
-        ),
-        panels.timeseries_count(
-            "Failed recovery attempts",
-            "Total number of failed reocovery attempts",
-            [
-                panels.target(f"sum({metric('recovery_failure_cnt')}) by (instance)",
-                              "{{instance}}")
-            ],
-            ["last"],
-        ),
-        panels.timeseries_latency(
-            "Recovery latency",
-            "Time spent in a successful recovery attempt",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({metric('recovery_latency_bucket')}[$__rate_interval])) by (le, instance))",
-                        f"recovery latency p{legend}" +
-                        " - {{instance}}",
+        outer_panels.row_collapsed(
+            "Recovery",
+        [
+            panels.timeseries_ops(
+                "Recovery Successful Rate",
+                "The rate of successful recovery attempts",
+                [
+                    panels.target(f"sum(rate({metric('recovery_latency_count')}[$__rate_interval])) by (instance)",
+                                "{{instance}}")
+                ],
+                ["last"],
+            ),
+            panels.timeseries_count(
+                "Failed recovery attempts",
+                "Total number of failed reocovery attempts",
+                [
+                    panels.target(f"sum({metric('recovery_failure_cnt')}) by (instance)",
+                                "{{instance}}")
+                ],
+                ["last"],
+            ),
+            panels.timeseries_latency(
+                "Recovery latency",
+                "Time spent in a successful recovery attempt",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({metric('recovery_latency_bucket')}[$__rate_interval])) by (le, instance))",
+                            f"recovery latency p{legend}" +
+                            " - {{instance}}",
+                        ),
+                        [50, 99, "max"],
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by (le) (rate({metric('recovery_latency_sum')}[$__rate_interval])) / sum by (le) (rate({metric('recovery_latency_count')}[$__rate_interval]))",
-                    "recovery latency avg",
-                ),
-            ],
-            ["last"],
-        )
+                    panels.target(
+                        f"sum by (le) (rate({metric('recovery_latency_sum')}[$__rate_interval])) / sum by (le) (rate({metric('recovery_latency_count')}[$__rate_interval]))",
+                        "recovery latency avg",
+                    ),
+                ],
+                ["last"],
+            )
+        ])
     ]
 
 
 def section_compaction(outer_panels):
+    successful_compaction_fiiler = "result='SUCCESS'"
+    failed_compaction_filter = "result!='SUCCESS'"
     panels = outer_panels.sub_panel()
     return [
         outer_panels.row_collapsed(
@@ -163,11 +177,21 @@ def section_compaction(outer_panels):
                     ],
                 ),
                 panels.timeseries_count(
-                    "Compaction Success & Failure Count",
+                    "Compaction Failure Count",
+                    "The number of compactions from one level to another level that have completed or failed",
+                    [
+                        panels.target(
+                            f"sum({metric('storage_level_compact_frequency', failed_compaction_filter)}) by (compactor, group, task_type, result)",
+                            "{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "Compaction Success Count",
                     "The number of compactions from one level to another level that have completed or failed",
                     [
                         panels.target(
-                            f"sum({metric('storage_level_compact_frequency')}) by (compactor, group, task_type, result)",
+                            f"sum({metric('storage_level_compact_frequency', successful_compaction_fiiler)}) by (compactor, group, task_type, result)",
                             "{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}",
                         ),
                     ],
@@ -474,6 +498,8 @@ def section_compaction(outer_panels):
 
 def section_object_storage(outer_panels):
     panels = outer_panels.sub_panel()
+    operation_rate_blacklist = "type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read'"
+    operation_duration_blacklist = "type!~'streaming_upload_write_bytes|streaming_read'"
     write_op_filter = "type=~'upload|delete'"
     read_op_filter = "type=~'read|readv|list|metadata'"
     request_cost_op1 = "type=~'read|streaming_read_start|delete'"
@@ -502,14 +528,14 @@ def section_object_storage(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('object_store_operation_latency_bucket')}[$__rate_interval])) by (le, type, job, instance))",
+                                f"histogram_quantile({quantile}, sum(rate({metric('object_store_operation_latency_bucket', operation_duration_blacklist)}[$__rate_interval])) by (le, type, job, instance))",
                                 "{{type}}" + f" p{legend}" +
                                 " - {{job}} @ {{instance}}",
                             ),
-                            [50, 90, 99, "max"],
+                            [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, type, job, instance)(rate({metric('object_store_operation_latency_sum')}[$__rate_interval])) / sum by(le, type, job, instance) (rate({metric('object_store_operation_latency_count')}[$__rate_interval]))",
+                            f"sum by(le, type, job, instance)(rate({metric('object_store_operation_latency_sum', operation_duration_blacklist)}[$__rate_interval])) / sum by(le, type, job, instance) (rate({metric('object_store_operation_latency_count')}[$__rate_interval]))",
                             "{{type}} avg - {{job}} @ {{instance}}",
                         ),
                     ],
@@ -519,7 +545,7 @@ def section_object_storage(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('object_store_operation_latency_count')}[$__rate_interval])) by (le, type, job, instance)",
+                            f"sum(rate({metric('object_store_operation_latency_count', operation_rate_blacklist)}[$__rate_interval])) by (le, type, job, instance)",
                             "{{type}} - {{job}} @ {{instance}}",
                         ),
                         panels.target(
@@ -541,7 +567,7 @@ def section_object_storage(outer_panels):
                             "{{type}}" + f" p{legend}" +
                             " - {{job}} @ {{instance}}",
                         ),
-                        [50, 90, 99, "max"],
+                        [50, 99, "max"],
                     ),
                 ),
                 panels.timeseries_ops(
@@ -608,237 +634,252 @@ def section_object_storage(outer_panels):
     ]
 
 
-def section_streaming(panels):
+def section_streaming(outer_panels):
+    panels = outer_panels.sub_panel()
     sink_filter = "executor_identity=~\".*SinkExecutor.*\""
     mv_filter = "executor_identity=~\".*MaterializeExecutor.*\""
     table_type_filter = "table_type=~\"MATERIALIZED_VIEW\""
     mv_throughput_query = f'sum(rate({metric("stream_executor_row_count", filter=mv_filter)}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group({metric("table_info", filter=table_type_filter)}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)'
     sink_throughput_query = f'sum(rate({metric("stream_executor_row_count", filter=sink_filter)}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group({metric("sink_info")}) by (actor_id, sink_name))) by (sink_name)'
     return [
-        panels.row("Streaming"),
-        panels.timeseries_rowsps(
-            "Source Throughput(rows/s)",
-            "The figure shows the number of rows read by each source per second.",
-            [
-                panels.target(
-                    f"rate({metric('stream_source_output_rows_counts')}[$__rate_interval])",
-                    "source={{source_name}} actor={{actor_id}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_rowsps(
-            "Source Throughput(rows/s) Per Partition",
-            "Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of "
-            "each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).",
-            [
-                panels.target(
-                    f"rate({metric('partition_input_count')}[$__rate_interval])",
-                    "actor={{actor_id}} source={{source_id}} partition={{partition}}",
-                )
-            ],
-        ),
-        panels.timeseries_bytesps(
-            "Source Throughput(MB/s)",
-            "The figure shows the number of bytes read by each source per second.",
-            [
-                panels.target(
-                    f"(sum by (source_id)(rate({metric('partition_input_bytes')}[$__rate_interval])))/(1000*1000)",
-                    "source={{source_id}}",
-                )
-            ],
-        ),
-        panels.timeseries_bytesps(
-            "Source Throughput(MB/s) Per Partition",
-            "Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of "
-            "each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).",
-            [
-                panels.target(
-                    f"(rate({metric('partition_input_bytes')}[$__rate_interval]))/(1000*1000)",
-                    "actor={{actor_id}} source={{source_id}} partition={{partition}}",
-                )
-            ],
-        ),
-        panels.timeseries_rowsps(
-            "Source Throughput(rows) per barrier",
-            "RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of "
-            "barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two "
-            "consecutive barriers.",
-            [
-                panels.target(
-                    f"rate({metric('stream_source_rows_per_barrier_counts')}[$__rate_interval])",
-                    "actor={{actor_id}} source={{source_id}} @ {{instance}}"
-                )
-            ]
-        ),
-        panels.timeseries_count(
-            "Source Upstream Status",
-            "Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.",
-            [
-                panels.target(
-                    f"{metric('source_status_is_up')}",
-                    "source_id={{source_id}}, source_name={{source_name}} @ {{instance}}"
-                )
-            ]
-        ),
-        panels.timeseries_ops(
-            "Source Split Change Events frequency(events/s)",
-            "Source Split Change Events frequency by source_id and actor_id",
-            [
-                panels.target(
-                    f"rate({metric('stream_source_split_change_event_count')}[$__rate_interval])",
-                    "source={{source_name}} actor={{actor_id}} @ {{instance}}"
-                )
-            ]
-        ),
-        panels.timeseries_count(
-            "Kafka Consumer Lag Size",
-            "Kafka Consumer Lag Size by source_id, partition and actor_id",
-            [
-                panels.target(
-                    f"{metric('high_watermark')}",
-                    "source={{source_id}} partition={{partition}}"
-                ),
-                panels.target(
-                    f"{metric('latest_message_id')}",
-                    "source={{source_id}} partition={{partition}} actor_id={{actor_id}}"
-                )
-            ]
-        ),
-        panels.timeseries_rowsps(
-            "Sink Throughput(rows/s)",
-            "The figure shows the number of rows output by each sink per second.",
-            [
-                panels.target(
-                    sink_throughput_query,
-                   "sink {{sink_name}}",
-                ),
-            ],
-        ),
+        outer_panels.row_collapsed(
+            "Streaming",
+        [
+            panels.timeseries_rowsps(
+                "Source Throughput(rows/s)",
+                "The figure shows the number of rows read by each source per second.",
+                [
+                    panels.target(
+                        f"rate({metric('stream_source_output_rows_counts')}[$__rate_interval])",
+                        "source={{source_name}} actor={{actor_id}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_rowsps(
+                "Source Throughput(rows/s) Per Partition",
+                "Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of "
+                "each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).",
+                [
+                    panels.target(
+                        f"rate({metric('partition_input_count')}[$__rate_interval])",
+                        "actor={{actor_id}} source={{source_id}} partition={{partition}}",
+                    )
+                ],
+            ),
+            panels.timeseries_bytesps(
+                "Source Throughput(MB/s)",
+                "The figure shows the number of bytes read by each source per second.",
+                [
+                    panels.target(
+                        f"(sum by (source_id)(rate({metric('partition_input_bytes')}[$__rate_interval])))/(1000*1000)",
+                        "source={{source_id}}",
+                    )
+                ],
+            ),
+            panels.timeseries_bytesps(
+                "Source Throughput(MB/s) Per Partition",
+                "Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of "
+                "each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).",
+                [
+                    panels.target(
+                        f"(rate({metric('partition_input_bytes')}[$__rate_interval]))/(1000*1000)",
+                        "actor={{actor_id}} source={{source_id}} partition={{partition}}",
+                    )
+                ],
+            ),
+            panels.timeseries_rowsps(
+                "Source Throughput(rows) per barrier",
+                "RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of "
+                "barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two "
+                "consecutive barriers.",
+                [
+                    panels.target(
+                        f"rate({metric('stream_source_rows_per_barrier_counts')}[$__rate_interval])",
+                        "actor={{actor_id}} source={{source_id}} @ {{instance}}"
+                    )
+                ]
+            ),
+            panels.timeseries_count(
+                "Source Upstream Status",
+                "Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.",
+                [
+                    panels.target(
+                        f"{metric('source_status_is_up')}",
+                        "source_id={{source_id}}, source_name={{source_name}} @ {{instance}}"
+                    )
+                ]
+            ),
+            panels.timeseries_ops(
+                "Source Split Change Events frequency(events/s)",
+                "Source Split Change Events frequency by source_id and actor_id",
+                [
+                    panels.target(
+                        f"rate({metric('stream_source_split_change_event_count')}[$__rate_interval])",
+                        "source={{source_name}} actor={{actor_id}} @ {{instance}}"
+                    )
+                ]
+            ),
+            panels.timeseries_count(
+                "Kafka Consumer Lag Size",
+                "Kafka Consumer Lag Size by source_id, partition and actor_id",
+                [
+                    panels.target(
+                        f"{metric('high_watermark')}",
+                        "source={{source_id}} partition={{partition}}"
+                    ),
+                    panels.target(
+                        f"{metric('latest_message_id')}",
+                        "source={{source_id}} partition={{partition}} actor_id={{actor_id}}"
+                    )
+                ]
+            ),
+            panels.timeseries_rowsps(
+                "Sink Throughput(rows/s)",
+                "The figure shows the number of rows output by each sink per second.",
+                [
+                    panels.target(
+                        sink_throughput_query,
+                    "sink {{sink_name}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_rowsps(
-            "Materialized View Throughput(rows/s)",
-            "The figure shows the number of rows written into each materialized view per second.",
-            [
-                panels.target(
-                   mv_throughput_query,
-                   "materialized view {{table_name}} table_id {{materialized_view_id}}",
-                ),
-            ],
-        ),
-        panels.timeseries_rowsps(
-            "Backfill Snapshot Read Throughput(rows)",
-            "Total number of rows that have been read from the backfill snapshot",
-            [
-                panels.target(
-                    f"rate({table_metric('stream_backfill_snapshot_read_row_count')}[$__rate_interval])",
-                    "table_id={{table_id}} actor={{actor_id}} @ {{instance}}"
-                ),
-            ],
-        ),
-        panels.timeseries_rowsps(
-            "Backfill Upstream Throughput(rows)",
-            "Total number of rows that have been output from the backfill upstream",
-            [
-                panels.target(
-                    f"rate({table_metric('stream_backfill_upstream_output_row_count')}[$__rate_interval])",
-                    "table_id={{table_id}} actor={{actor_id}} @ {{instance}}"
-                ),
-            ],
-        ),
-        panels.timeseries_count(
-            "Barrier Number",
-            "The number of barriers that have been ingested but not completely processed. This metric reflects the "
-            "current level of congestion within the system.",
-            [
-                panels.target(f"{metric('all_barrier_nums')}", "all_barrier"),
-                panels.target(
-                    f"{metric('in_flight_barrier_nums')}", "in_flight_barrier"),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Barrier Send Latency",
-            "The duration between the time point when the scheduled barrier needs to be sent and the time point when "
-            "the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal "
-            "congestion.",
-            quantile(
-                lambda quantile, legend: panels.target(
-                    f"histogram_quantile({quantile}, sum(rate({metric('meta_barrier_send_duration_seconds_bucket')}[$__rate_interval])) by (le))",
-                    f"barrier_send_latency_p{legend}",
-                ),
-                [50, 90, 99, 999, "max"],
-            ) + [
-                panels.target(
-                    f"rate({metric('meta_barrier_send_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_send_duration_seconds_count')}[$__rate_interval])",
-                    "barrier_send_latency_avg",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Barrier Latency",
-            "The time that the data between two consecutive barriers gets fully processed, i.e. the computation "
-            "results are made durable into materialized views or sink to external systems. This metric shows to users "
-            "the freshness of materialized views.",
-            quantile(
-                lambda quantile, legend: panels.target(
-                    f"histogram_quantile({quantile}, sum(rate({metric('meta_barrier_duration_seconds_bucket')}[$__rate_interval])) by (le))",
-                    f"barrier_latency_p{legend}",
-                ),
-                [50, 90, 99, 999, "max"],
-            ) + [
-                panels.target(
-                    f"rate({metric('meta_barrier_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_duration_seconds_count')}[$__rate_interval])",
-                    "barrier_latency_avg",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Barrier In-Flight Latency",
-            "",
-            quantile(
-                lambda quantile, legend: panels.target(
-                    f"histogram_quantile({quantile}, sum(rate({metric('stream_barrier_inflight_duration_seconds_bucket')}[$__rate_interval])) by (le))",
-                    f"barrier_inflight_latency_p{legend}",
-                ),
-                [50, 90, 99, 999, "max"],
-            ) + [
-                panels.target(
-                    f"max(sum by(le, instance)(rate({metric('stream_barrier_inflight_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, instance)(rate({metric('stream_barrier_inflight_duration_seconds_count')}[$__rate_interval])))",
-                    "barrier_inflight_latency_avg",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Barrier Sync Latency",
-            "",
-            quantile(
-                lambda quantile, legend: panels.target(
-                    f"histogram_quantile({quantile}, sum(rate({metric('stream_barrier_sync_storage_duration_seconds_bucket')}[$__rate_interval])) by (le,instance))",
-                    f"barrier_sync_latency_p{legend}" + " - {{instance}}",
-                ),
-                [50, 90, 99, 999, "max"],
-            ) + [
-                panels.target(
-                    f"sum by(le, instance)(rate({metric('stream_barrier_sync_storage_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, instance)(rate({metric('stream_barrier_sync_storage_duration_seconds_count')}[$__rate_interval]))",
-                    "barrier_sync_latency_avg - {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Barrier Wait Commit Latency",
-            "",
-            quantile(
-                lambda quantile, legend: panels.target(
-                    f"histogram_quantile({quantile}, sum(rate({metric('meta_barrier_wait_commit_duration_seconds_bucket')}[$__rate_interval])) by (le))",
-                    f"barrier_wait_commit_latency_p{legend}",
-                ),
-                [50, 90, 99, 999, "max"],
-            ) + [
-                panels.target(
-                    f"rate({metric('meta_barrier_wait_commit_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_wait_commit_duration_seconds_count')}[$__rate_interval])",
-                    "barrier_wait_commit_avg",
-                ),
-            ],
-        ),
+            panels.timeseries_rowsps(
+                "Materialized View Throughput(rows/s)",
+                "The figure shows the number of rows written into each materialized view per second.",
+                [
+                    panels.target(
+                    mv_throughput_query,
+                    "materialized view {{table_name}} table_id {{materialized_view_id}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_rowsps(
+                "Backfill Snapshot Read Throughput(rows)",
+                "Total number of rows that have been read from the backfill snapshot",
+                [
+                    panels.target(
+                        f"rate({table_metric('stream_backfill_snapshot_read_row_count')}[$__rate_interval])",
+                        "table_id={{table_id}} actor={{actor_id}} @ {{instance}}"
+                    ),
+                ],
+            ),
+            panels.timeseries_rowsps(
+                "Backfill Upstream Throughput(rows)",
+                "Total number of rows that have been output from the backfill upstream",
+                [
+                    panels.target(
+                        f"rate({table_metric('stream_backfill_upstream_output_row_count')}[$__rate_interval])",
+                        "table_id={{table_id}} actor={{actor_id}} @ {{instance}}"
+                    ),
+                ],
+            ),
+            panels.timeseries_count(
+                "Barrier Number",
+                "The number of barriers that have been ingested but not completely processed. This metric reflects the "
+                "current level of congestion within the system.",
+                [
+                    panels.target(f"{metric('all_barrier_nums')}", "all_barrier"),
+                    panels.target(
+                        f"{metric('in_flight_barrier_nums')}", "in_flight_barrier"),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Barrier Send Latency",
+                "The duration between the time point when the scheduled barrier needs to be sent and the time point when "
+                "the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal "
+                "congestion.",
+                quantile(
+                    lambda quantile, legend: panels.target(
+                        f"histogram_quantile({quantile}, sum(rate({metric('meta_barrier_send_duration_seconds_bucket')}[$__rate_interval])) by (le))",
+                        f"barrier_send_latency_p{legend}",
+                    ),
+                    [50, 90, 99, 999, "max"],
+                ) + [
+                    panels.target(
+                        f"rate({metric('meta_barrier_send_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_send_duration_seconds_count')}[$__rate_interval])",
+                        "barrier_send_latency_avg",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Barrier Latency",
+                "The time that the data between two consecutive barriers gets fully processed, i.e. the computation "
+                "results are made durable into materialized views or sink to external systems. This metric shows to users "
+                "the freshness of materialized views.",
+                quantile(
+                    lambda quantile, legend: panels.target(
+                        f"histogram_quantile({quantile}, sum(rate({metric('meta_barrier_duration_seconds_bucket')}[$__rate_interval])) by (le))",
+                        f"barrier_latency_p{legend}",
+                    ),
+                    [50, 90, 99, 999, "max"],
+                ) + [
+                    panels.target(
+                        f"rate({metric('meta_barrier_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_duration_seconds_count')}[$__rate_interval])",
+                        "barrier_latency_avg",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Barrier In-Flight Latency",
+                "",
+                quantile(
+                    lambda quantile, legend: panels.target(
+                        f"histogram_quantile({quantile}, sum(rate({metric('stream_barrier_inflight_duration_seconds_bucket')}[$__rate_interval])) by (le))",
+                        f"barrier_inflight_latency_p{legend}",
+                    ),
+                    [50, 90, 99, 999, "max"],
+                ) + [
+                    panels.target(
+                        f"max(sum by(le, instance)(rate({metric('stream_barrier_inflight_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, instance)(rate({metric('stream_barrier_inflight_duration_seconds_count')}[$__rate_interval])))",
+                        "barrier_inflight_latency_avg",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Barrier Sync Latency",
+                "",
+                quantile(
+                    lambda quantile, legend: panels.target(
+                        f"histogram_quantile({quantile}, sum(rate({metric('stream_barrier_sync_storage_duration_seconds_bucket')}[$__rate_interval])) by (le,instance))",
+                        f"barrier_sync_latency_p{legend}" + " - {{instance}}",
+                    ),
+                    [50, 90, 99, 999, "max"],
+                ) + [
+                    panels.target(
+                        f"sum by(le, instance)(rate({metric('stream_barrier_sync_storage_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, instance)(rate({metric('stream_barrier_sync_storage_duration_seconds_count')}[$__rate_interval]))",
+                        "barrier_sync_latency_avg - {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Barrier Wait Commit Latency",
+                "",
+                quantile(
+                    lambda quantile, legend: panels.target(
+                        f"histogram_quantile({quantile}, sum(rate({metric('meta_barrier_wait_commit_duration_seconds_bucket')}[$__rate_interval])) by (le))",
+                        f"barrier_wait_commit_latency_p{legend}",
+                    ),
+                    [50, 90, 99, 999, "max"],
+                ) + [
+                    panels.target(
+                        f"rate({metric('meta_barrier_wait_commit_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_wait_commit_duration_seconds_count')}[$__rate_interval])",
+                        "barrier_wait_commit_avg",
+                    ),
+                ],
+            ),
+            panels.timeseries_ops(
+                "Earliest In-Flight Barrier Progress",
+                "The number of actors that have processed the earliest in-flight barriers per second. "
+                "This metric helps users to detect potential congestion or stuck in the system.",
+                [
+                    panels.target(
+                        f"rate({metric('stream_barrier_manager_progress')}[$__rate_interval])",
+                        "{{instance}}",
+                    ),
+                ],
+            ),
+        ])
     ]
 
 
@@ -1048,6 +1089,10 @@ def section_streaming_actors(outer_panels):
                             "materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}}  {{instance}}",
                         ),
 
+                        panels.target(
+                            f"(sum(rate({metric('stream_over_window_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_over_window_cache_lookup_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            "Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ",
+                        ),
                     ],
                 ),
                 panels.timeseries_actor_latency(
@@ -1056,14 +1101,14 @@ def section_streaming_actors(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('stream_join_barrier_align_duration_bucket')}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))",
-                                f"p{legend} {{{{actor_id}}}}.{{{{wait_side}}}} - {{{{job}}}} @ {{{{instance}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({metric('stream_join_barrier_align_duration_bucket')}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))",
+                                f"p{legend} - fragment {{{{fragment_id}}}} {{{{wait_side}}}} - {{{{job}}}} @ {{{{instance}}}}",
                             ),
                             [90, 99, 999, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, actor_id, wait_side, job, instance)(rate({metric('stream_join_barrier_align_duration_sum')}[$__rate_interval])) / sum by(le,actor_id,wait_side,job,instance) (rate({metric('stream_join_barrier_align_duration_count')}[$__rate_interval]))",
-                            "avg {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}",
+                            f"sum by(le, fragment_id, wait_side, job, instance)(rate({metric('stream_join_barrier_align_duration_sum')}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate({metric('stream_join_barrier_align_duration_count')}[$__rate_interval]))",
+                            "avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}",
                         ),
                     ],
                 ),
@@ -1120,14 +1165,14 @@ def section_streaming_actors(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('stream_join_matched_join_keys_bucket')}[$__rate_interval])) by (le, actor_id, table_id, job, instance))",
-                                f"p{legend} - actor_id {{{{actor_id}}}} table_id {{{{table_id}}}} - {{{{job}}}} @ {{{{instance}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({metric('stream_join_matched_join_keys_bucket')}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))",
+                                f"p{legend} - fragment {{{{fragment_id}}}} table_id {{{{table_id}}}} - {{{{job}}}} @ {{{{instance}}}}",
                             ),
                             [90, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, job, instance, actor_id, table_id) (rate({metric('stream_join_matched_join_keys_sum')}[$__rate_interval])) / sum by(le, job, instance, actor_id, table_id) (rate({table_metric('stream_join_matched_join_keys_count')}[$__rate_interval]))",
-                            "avg - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}",
+                            f"sum by(le, job, instance, actor_id, table_id) (rate({metric('stream_join_matched_join_keys_sum')}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate({table_metric('stream_join_matched_join_keys_count')}[$__rate_interval]))",
+                            "avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}",
                         ),
                     ],
                 ),
@@ -1220,6 +1265,25 @@ def section_streaming_actors(outer_panels):
 
                     ],
                 ),
+
+                panels.timeseries_actor_ops(
+                    "Over Window Executor Cache",
+                    "",
+                    [
+                        panels.target(
+                            f"rate({table_metric('stream_over_window_cached_entry_count')}[$__rate_interval])",
+                            "cached entry count - table {{table_id}} - actor {{actor_id}}   {{instance}}",
+                        ),
+                        panels.target(
+                            f"rate({table_metric('stream_over_window_cache_lookup_count')}[$__rate_interval])",
+                            "cache lookup count - table {{table_id}} - actor {{actor_id}}   {{instance}}",
+                        ),
+                        panels.target(
+                            f"rate({table_metric('stream_over_window_cache_miss_count')}[$__rate_interval])",
+                            "cache miss count - table {{table_id}} - actor {{actor_id}}   {{instance}}",
+                        ),
+                    ]
+                ),
             ],
         )
     ]
@@ -1443,6 +1507,16 @@ def section_streaming_errors(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_count(
+                    "Source Reader Errors by Type",
+                    "",
+                    [
+                        panels.target(
+                            f"sum({metric('user_source_reader_error_count')}) by (error_type, error_msg, actor_id, source_id, executor_name)",
+                            "{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})",
+                        ),
+                    ],
+                ),
             ],
         ),
     ]
@@ -1494,6 +1568,24 @@ def section_batch(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_latency(
+                    "Row SeqScan Next Duration",
+                    "",
+                    [
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('batch_row_seq_scan_next_duration_bucket')}[$__rate_interval])) by (le, job, instance))",
+                                f"row_seq_scan next p{legend}" +
+                                " - {{job}} @ {{instance}}",
+                            ),
+                            [50, 99, "max"],
+                        ),
+                        panels.target(
+                            f"sum by(le, job, instance) (rate({metric('batch_row_seq_scan_next_duration_sum')}[$__rate_interval])) / sum by(le, job, instance) (rate({metric('batch_row_seq_scan_next_duration_count')}[$__rate_interval]))",
+                            "row_seq_scan next avg - {{job}} @ {{instance}}",
+                        ),
+                    ],
+                ),
             ],
         ),
     ]
@@ -1603,7 +1695,8 @@ def section_frontend(outer_panels):
     ]
 
 
-def section_hummock(panels):
+def section_hummock_read(outer_panels):
+    panels = outer_panels.sub_panel()
     meta_miss_filter = "type='meta_miss'"
     meta_total_filter = "type='meta_total'"
     data_miss_filter = "type='data_miss'"
@@ -1611,553 +1704,485 @@ def section_hummock(panels):
     file_cache_get_filter = "op='get'"
 
     return [
-        panels.row("Hummock"),
-        panels.timeseries_latency(
-            "Build and Sync Sstable Duration",
-            "Histogram of time spent on compacting shared buffer to remote storage.",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({metric('state_store_sync_duration_bucket')}[$__rate_interval])) by (le, job, instance))",
-                        f"p{legend}" + " - {{job}} @ {{instance}}",
-                    ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance) (rate({metric('state_store_sync_duration_sum')}[$__rate_interval])) / sum by(le, job, instance) (rate({metric('state_store_sync_duration_count')}[$__rate_interval]))",
-                    "avg - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_ops(
-            "Cache Ops",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({table_metric('state_store_sst_store_block_request_counts')}[$__rate_interval])) by (job, instance, table_id, type)",
-                    "{{table_id}} @ {{type}} - {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum(rate({metric('state_store_sst_store_block_request_counts', meta_miss_filter)}[$__rate_interval])) by (job, instance, type)",
-                    "total_meta_miss_count - {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum(rate({metric('sstable_preload_io_count')}[$__rate_interval])) ",
-                    "preload iops",
-                ),
-            ],
-        ),
-        panels.timeseries_ops(
-            "File Cache Ops",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({metric('foyer_storage_latency_count')}[$__rate_interval])) by (op, extra, instance)",
-                    "file cache {{op}} {{extra}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_ops(
-            "Read Ops",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({table_metric('state_store_get_duration_count')}[$__rate_interval])) by (job,instanc,table_id)",
-                    "get - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum(rate({table_metric('state_store_range_reverse_scan_duration_count')}[$__rate_interval])) by (job,instance)",
-                    "backward scan - {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum(rate({table_metric('state_store_get_shared_buffer_hit_counts')}[$__rate_interval])) by (job,instance,table_id)",
-                    "shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum(rate({table_metric('state_store_iter_in_process_counts')}[$__rate_interval])) by(job,instance,table_id)",
-                    "iter - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Read Duration - Get",
-            "Histogram of the latency of Get operations that have been issued to the state store.",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_get_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"p{legend}" + " - {{table_id}} @ {{job}} @ {{instance}}",
-                    ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_get_duration_sum')}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate({table_metric('state_store_get_duration_count')}[$__rate_interval]))",
-                    "avg - {{table_id}} {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Read Duration - Iter",
-            "Histogram of the time spent on iterator initialization."
-            "Histogram of the time spent on iterator scanning.",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_init_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"create_iter_time p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+        outer_panels.row_collapsed(
+            "Hummock (Read)",
+        [
+            panels.timeseries_ops(
+                "Cache Ops",
+                "",
+                [
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_sst_store_block_request_counts')}[$__rate_interval])) by (job, instance, table_id, type)",
+                        "{{table_id}} @ {{type}} - {{job}} @ {{instance}}",
                     ),
-                    [90, 99, 999, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance)(rate({metric('state_store_iter_init_duration_sum')}[$__rate_interval])) / sum by(le, job,instance) (rate({metric('state_store_iter_init_duration_count')}[$__rate_interval]))",
-                    "create_iter_time avg - {{job}} @ {{instance}}",
-                ),
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_scan_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"pure_scan_time p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+                    panels.target(
+                        f"sum(rate({metric('state_store_sst_store_block_request_counts', meta_miss_filter)}[$__rate_interval])) by (job, instance, type)",
+                        "total_meta_miss_count - {{job}} @ {{instance}}",
                     ),
-                    [90, 99, 999, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance)(rate({metric('state_store_iter_scan_duration_sum')}[$__rate_interval])) / sum by(le, job,instance) (rate({metric('state_store_iter_scan_duration_count')}[$__rate_interval]))",
-                    "pure_scan_time avg - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes(
-            "Read Item Size - Get",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_get_key_size_bucket')}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile({quantile}, sum(rate({table_metric('state_store_get_value_size_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"p{legend} - {{{{table_id}}}} {{{{job}}}} @ {{{{instance}}}}",
+                ],
+            ),
+
+            panels.timeseries_bytes(
+                "Cache Size",
+                "Hummock has three parts of memory usage: 1. Meta Cache 2. Block Cache"
+                "This metric shows the real memory usage of each of these three caches.",
+                [
+                    panels.target(
+                        f"avg({metric('state_store_meta_cache_size')}) by (job,instance)",
+                        "meta cache - {{job}} @ {{instance}}",
                     ),
-                    [90, 99, 999, "max"],
-                ),
-            ],
-        ),
-        panels.timeseries_bytes(
-            "Read Item Size - Iter",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_size_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+                    panels.target(
+                        f"avg({metric('state_store_block_cache_size')}) by (job,instance)",
+                        "data cache - {{job}} @ {{instance}}",
                     ),
-                    [90, 99, 999, "max"],
-                ),
-            ],
-        ),
+                ],
+            ),
 
-        panels.timeseries_bytes(
-            "Materialized View Read Size",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f'sum(histogram_quantile({quantile}, sum(rate({metric("state_store_iter_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group({metric("table_info")}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile({quantile}, sum(rate({metric("state_store_get_key_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile({quantile}, sum(rate({metric("state_store_get_value_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group({metric("table_info")}) by (materialized_view_id, table_id))) by (materialized_view_id)',
-                        f"read p{legend} - materialized view {{{{materialized_view_id}}}}"
+            panels.timeseries_percentage(
+                "Cache Miss Rate",
+                "",
+                [
+                    panels.target(
+                        f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_miss_filter)}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_total_filter)}[$__rate_interval])) by (job,instance,table_id))",
+                        "meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}",
                     ),
-                    [90, 99, "max"],
-                ),
-            ],
-        ),
+                    panels.target(
+                        f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', data_miss_filter)}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', data_total_filter)}[$__rate_interval])) by (job,instance,table_id))",
+                        "block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_bytes(
-            "Materialized View Write Size",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f'sum(histogram_quantile({quantile}, sum(rate({metric("state_store_write_batch_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group({metric("table_info")}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)',
-                        f"write p{legend} - materialized view {{{{materialized_view_id}}}}"
+            panels.timeseries_ops(
+                "Iter keys flow",
+                "",
+                [
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_iter_scan_key_counts')}[$__rate_interval])) by (instance, type, table_id)",
+                        "iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ",
                     ),
-                    [90, 99, "max"],
-                ),
-            ],
-        ),
+                ],
+            ),
+            panels.timeseries_count(
+                "Read Merged SSTs",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_merge_sstable_counts_bucket')}[$__rate_interval])) by (le, job, table_id, type))",
+                            f"# merged ssts p{legend}" +
+                            " - {{table_id}} @ {{job}} @ {{type}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_sum')}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_count')}[$__rate_interval]))",
+                        "# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
 
+            panels.timeseries_latency(
+                "Read Duration - Get",
+                "Histogram of the latency of Get operations that have been issued to the state store.",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_get_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"p{legend}" + " - {{table_id}} @ {{job}} @ {{instance}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_get_duration_sum')}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate({table_metric('state_store_get_duration_count')}[$__rate_interval]))",
+                        "avg - {{table_id}} {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Read Duration - Iter",
+                "Histogram of the time spent on iterator initialization."
+                "Histogram of the time spent on iterator scanning.",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_init_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"create_iter_time p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance)(rate({metric('state_store_iter_init_duration_sum')}[$__rate_interval])) / sum by(le, job,instance) (rate({metric('state_store_iter_init_duration_count')}[$__rate_interval]))",
+                        "create_iter_time avg - {{job}} @ {{instance}}",
+                    ),
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_scan_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"pure_scan_time p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance)(rate({metric('state_store_iter_scan_duration_sum')}[$__rate_interval])) / sum by(le, job,instance) (rate({metric('state_store_iter_scan_duration_count')}[$__rate_interval]))",
+                        "pure_scan_time avg - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Read Item Count - Iter",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_item_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+            panels.timeseries_ops(
+                "Bloom Filter Ops",
+                "",
+                [
+                    panels.target(
+                        f"sum(irate({table_metric('state_store_read_req_positive_but_non_exist_counts')}[$__rate_interval])) by (table_id,type)",
+                        "bloom filter false positive count  - {{table_id}} - {{type}}",
                     ),
-                    [90, 99, 999, "max"],
-                ),
-            ],
-        ),
-        panels.timeseries_bytes_per_sec(
-            "Read Throughput - Get",
-            "The size of a single key-value pair when reading by operation Get."
-            "Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not "
-            "exist in the storage, the size of key is counted into this metric and the size of value is 0.",
-            [
-                panels.target(
-                    f"sum(rate({metric('state_store_get_key_size_sum')}[$__rate_interval])) by(job, instance) + sum(rate({metric('state_store_get_value_size_sum')}[$__rate_interval])) by(job, instance)",
-                    "{{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes_per_sec(
-            "Read Throughput - Iter",
-            "The size of all the key-value paris when reading by operation Iter."
-            "Operation Iter scans a range of key-value pairs.",
-            [
-                panels.target(
-                    f"sum(rate({metric('state_store_iter_size_sum')}[$__rate_interval])) by(job, instance)",
-                    "{{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Read Duration - MayExist",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_may_exist_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"p{legend}" + " - {{table_id}} @ {{job}} @ {{instance}}",
+                    panels.target(
+                        f"sum(irate({table_metric('state_store_read_req_bloom_filter_positive_counts')}[$__rate_interval])) by (table_id,type)",
+                        "bloom filter positive count - {{table_id}} - {{type}}",
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_may_exist_duration_sum')}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate({table_metric('state_store_may_exist_duration_count')}[$__rate_interval]))",
-                    "avg - {{table_id}} {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_ops(
-            "Read Bloom Filter",
-            "",
-            [
-                panels.target(
-                    f"sum(irate({table_metric('state_store_read_req_positive_but_non_exist_counts')}[$__rate_interval])) by (table_id,type)",
-                    "bloom filter false positive count  - {{table_id}} - {{type}}",
-                ),
-                panels.target(
-                    f"sum(irate({table_metric('state_store_read_req_bloom_filter_positive_counts')}[$__rate_interval])) by (table_id,type)",
-                    "bloom filter miss count - {{table_id}} - {{type}}",
-                ),
-                panels.target(
-                    f"sum(irate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type)",
-                    "bloom filter check count- {{table_id}} - {{type}}",
-                ),
-            ],
-        ),
-        panels.timeseries_ops(
-            "Iter keys flow",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({table_metric('state_store_iter_scan_key_counts')}[$__rate_interval])) by (instance, type, table_id)",
-                    "iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ",
-                ),
-            ],
-        ),
-        panels.timeseries_percentage(
-            "Cache Miss Rate",
-            "",
-            [
-                panels.target(
-                    f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_miss_filter)}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_total_filter)}[$__rate_interval])) by (job,instance,table_id))",
-                    "meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', data_miss_filter)}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', data_total_filter)}[$__rate_interval])) by (job,instance,table_id))",
-                    "block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"(sum(rate({metric('file_cache_miss')}[$__rate_interval])) by (instance)) / (sum(rate({metric('file_cache_latency_count', file_cache_get_filter)}[$__rate_interval])) by (instance))",
-                    "file cache miss rate @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_percentage(
-            "Bloom-Filter Miss Rate",
-            "Positive / Total",
-            [
-                panels.target(
-                    f"(sum(rate({table_metric('state_store_read_req_bloom_filter_positive_counts')}[$__rate_interval])) by (table_id,type)) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type))",
-                    "bloom filter miss rate - {{table_id}} - {{type}}",
-                ),
-            ],
-        ),
-        panels.timeseries_percentage(
-            "Read Request Bloom-Filter False-Positive Rate",
-            "False-Positive / Total",
-            [
-                panels.target(
-                    f"(((sum(rate({table_metric('state_store_read_req_positive_but_non_exist_counts')}[$__rate_interval])) by (table_id,type))) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type)))",
-                    "read req bloom filter false positive rate - {{table_id}} - {{type}}",
-                ),
-            ],
-        ),
-        panels.timeseries_count(
-            "Read Merged SSTs",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_merge_sstable_counts_bucket')}[$__rate_interval])) by (le, job, table_id, type))",
-                        f"# merged ssts p{legend}" +
-                        " - {{table_id}} @ {{job}} @ {{type}}",
+                    panels.target(
+                        f"sum(irate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type)",
+                        "bloom filter check count- {{table_id}} - {{type}}",
                     ),
-                    [90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_sum')}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_count')}[$__rate_interval]))",
-                    "# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Uploader - Tasks Count",
-            "",
-            [
-                panels.target(
-                    f"sum(irate({table_metric('state_store_merge_imm_task_counts')}[$__rate_interval])) by (job,instance,table_id)",
-                    "merge imm tasks - {{table_id}} @ {{instance}} ",
-                ),
-                panels.target(
-                    f"sum(irate({metric('state_store_spill_task_counts')}[$__rate_interval])) by (job,instance,uploader_stage)",
-                    "Uploader spill tasks - {{uploader_stage}} @ {{instance}} ",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes(
-            "Uploader - Task Size",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({table_metric('state_store_merge_imm_memory_sz')}[$__rate_interval])) by (job,instance,table_id)",
-                    "Merging tasks memory size - {{table_id}} @ {{instance}} ",
-                ),
-                panels.target(
-                    f"sum(rate({metric('state_store_spill_task_size')}[$__rate_interval])) by (job,instance,uploader_stage)",
-                    "Uploading tasks size - {{uploader_stage}} @ {{instance}} ",
-                ),
-            ],
-        ),
+            panels.timeseries_percentage(
+                "Bloom Filter Positive Rate",
+                "Positive / Total",
+                [
+                    panels.target(
+                        f"(sum(rate({table_metric('state_store_read_req_bloom_filter_positive_counts')}[$__rate_interval])) by (table_id,type)) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type))",
+                        "bloom filter positive rate - {{table_id}} - {{type}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_percentage(
+                "Bloom Filter False-Positive Rate",
+                "False-Positive / Total",
+                [
+                    panels.target(
+                        f"(((sum(rate({table_metric('state_store_read_req_positive_but_non_exist_counts')}[$__rate_interval])) by (table_id,type))) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type)))",
+                        "read req bloom filter false positive rate - {{table_id}} - {{type}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_ops(
-            "Write Ops",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({table_metric('state_store_write_batch_duration_count')}[$__rate_interval])) by (job,instance,table_id)",
-                    "write batch - {{table_id}} @ {{job}} @ {{instance}} ",
-                ),
-                panels.target(
-                    f"sum(rate({metric('state_store_sync_duration_count')}[$__rate_interval])) by (job,instance)",
-                    "l0 - {{job}} @ {{instance}} ",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Write Duration",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_write_batch_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"write to shared_buffer p{legend}" +
-                        " - {{table_id}} @ {{job}} @ {{instance}}",
+            panels.timeseries_count(
+                "Slow Fetch Meta Unhits",
+                "",
+                [
+                    panels.target(
+                        f"{metric('state_store_iter_slow_fetch_meta_cache_unhits')}",
+                        "",
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_write_batch_duration_sum')}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate({table_metric('state_store_write_batch_duration_count')}[$__rate_interval]))",
-                    "write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({metric('state_store_write_shared_buffer_sync_time_bucket')}[$__rate_interval])) by (le, job, instance))",
-                        f"write to object_store p{legend}" +
-                        " - {{job}} @ {{instance}}",
+                ],
+            ),
+
+            panels.timeseries_ops(
+                "Read Ops",
+                "",
+                [
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_get_duration_count')}[$__rate_interval])) by (job,instanc,table_id)",
+                        "get - {{table_id}} @ {{job}} @ {{instance}}",
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance)(rate({metric('state_store_write_shared_buffer_sync_time_sum')}[$__rate_interval]))  / sum by(le, job, instance)(rate({metric('state_store_write_shared_buffer_sync_time_count')}[$__rate_interval]))",
-                    "write to object_store - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_ops(
-            "Write Item Count",
-            "",
-            [
-                panels.target(
-                    f"sum(irate({table_metric('state_store_write_batch_tuple_counts')}[$__rate_interval])) by (job,instance,table_id)",
-                    "write_batch_kv_pair_count - {{table_id}} @ {{instance}} ",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes_per_sec(
-            "Write Throughput",
-            "",
-            [
-                panels.target(
-                    f"sum(rate({table_metric('state_store_write_batch_size_sum')}[$__rate_interval]))by(job,instance,table_id) / sum(rate({table_metric('state_store_write_batch_size_count')}[$__rate_interval]))by(job,instance,table_id)",
-                    "shared_buffer - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum(rate({metric('compactor_shared_buffer_to_sstable_size_sum')}[$__rate_interval]))by(job,instance) / sum(rate({metric('compactor_shared_buffer_to_sstable_size_count')}[$__rate_interval]))by(job,instance)",
-                    "sync - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes(
-            "Mem Table Size (Max)",
-            "This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.",
-            [
-                panels.target(
-                    f"histogram_quantile(1.0, sum(rate({metric('state_store_write_batch_size_bucket')}[$__rate_interval])) by (le, table_id, job, instance))",
-                    "pmax - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes(
-            "Checkpoint Sync Size",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({metric('state_store_sync_size_bucket')}[$__rate_interval])) by (le, job, instance))",
-                        f"p{legend}" + " - {{job}} @ {{instance}}",
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_get_shared_buffer_hit_counts')}[$__rate_interval])) by (job,instance,table_id)",
+                        "shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}",
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance) (rate({metric('state_store_sync_size_sum')}[$__rate_interval])) / sum by(le, job, instance) (rate({metric('state_store_sync_size_count')}[$__rate_interval]))",
-                    "avg - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_bytes(
-            "Cache Size",
-            "Hummock has three parts of memory usage: 1. Meta Cache 2. Block Cache 3. Uploader."
-            "This metric shows the real memory usage of each of these three caches.",
-            [
-                panels.target(
-                    f"avg({metric('state_store_meta_cache_size')}) by (job,instance)",
-                    "meta cache - {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"avg({metric('state_store_block_cache_size')}) by (job,instance)",
-                    "data cache - {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum({metric('uploading_memory_size')}) by (job,instance)",
-                    "uploading memory - {{job}} @ {{instance}}",
-                ),
-                panels.target(
-                    f"sum({metric('state_store_uploader_uploading_task_size')}) by (job,instance)",
-                    "uploading task size - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
-        panels.timeseries_latency(
-            "Row SeqScan Next Duration",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({metric('batch_row_seq_scan_next_duration_bucket')}[$__rate_interval])) by (le, job, instance))",
-                        f"row_seq_scan next p{legend}" +
-                        " - {{job}} @ {{instance}}",
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_iter_in_process_counts')}[$__rate_interval])) by(job,instance,table_id)",
+                        "iter - {{table_id}} @ {{job}} @ {{instance}}",
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance) (rate({metric('batch_row_seq_scan_next_duration_sum')}[$__rate_interval])) / sum by(le, job, instance) (rate({metric('batch_row_seq_scan_next_duration_count')}[$__rate_interval]))",
-                    "row_seq_scan next avg - {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
+                ],
+            ),
+            panels.timeseries_bytes(
+                "Read Item Size - Get",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_get_key_size_bucket')}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile({quantile}, sum(rate({table_metric('state_store_get_value_size_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"p{legend} - {{{{table_id}}}} {{{{job}}}} @ {{{{instance}}}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes(
+                "Read Item Size - Iter",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_size_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                ],
+            ),
 
-        panels.timeseries_latency(
-            "Fetch Meta Duration",
-            "",
-            [
-                *quantile(
-                    lambda quantile, legend: panels.target(
-                        f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_fetch_meta_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
-                        f"fetch_meta_duration p{legend}" +
-                        " - {{table_id}} @ {{job}} @ {{instance}}",
+            panels.timeseries_bytes(
+                "Materialized View Read Size",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f'sum(histogram_quantile({quantile}, sum(rate({metric("state_store_iter_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group({metric("table_info")}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile({quantile}, sum(rate({metric("state_store_get_key_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile({quantile}, sum(rate({metric("state_store_get_value_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group({metric("table_info")}) by (materialized_view_id, table_id))) by (materialized_view_id)',
+                            f"read p{legend} - materialized view {{{{materialized_view_id}}}}"
+                        ),
+                        [50, 99, "max"],
                     ),
-                    [50, 90, 99, "max"],
-                ),
-                panels.target(
-                    f"sum by(le, job, instance, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_sum')}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_count')}[$__rate_interval]))",
-                    "fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}",
-                ),
-            ],
-        ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Fetch Meta Unhits",
-            "",
-            [
-                panels.target(
-                    f"{metric('state_store_iter_fetch_meta_cache_unhits')}",
-                    "",
-                ),
-            ],
-        ),
+            panels.timeseries_count(
+                "Read Item Count - Iter",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_item_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"p{legend} - {{{{table_id}}}} @ {{{{job}}}} @ {{{{instance}}}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes_per_sec(
+                "Read Throughput - Get",
+                "The size of a single key-value pair when reading by operation Get."
+                "Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not "
+                "exist in the storage, the size of key is counted into this metric and the size of value is 0.",
+                [
+                    panels.target(
+                        f"sum(rate({metric('state_store_get_key_size_sum')}[$__rate_interval])) by(job, instance) + sum(rate({metric('state_store_get_value_size_sum')}[$__rate_interval])) by(job, instance)",
+                        "{{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes_per_sec(
+                "Read Throughput - Iter",
+                "The size of all the key-value paris when reading by operation Iter."
+                "Operation Iter scans a range of key-value pairs.",
+                [
+                    panels.target(
+                        f"sum(rate({metric('state_store_iter_size_sum')}[$__rate_interval])) by(job, instance)",
+                        "{{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Slow Fetch Meta Unhits",
-            "",
-            [
-                panels.target(
-                    f"{metric('state_store_iter_slow_fetch_meta_cache_unhits')}",
-                    "",
-                ),
-            ],
-        ),
+            panels.timeseries_latency(
+                "Fetch Meta Duration",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_fetch_meta_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"fetch_meta_duration p{legend}" +
+                            " - {{table_id}} @ {{job}} @ {{instance}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_sum')}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_count')}[$__rate_interval]))",
+                        "fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Move State Table Count",
-            "The times of move_state_table occurs",
-            [
-                panels.target(
-                    f"sum({table_metric('storage_move_state_table_count')}) by (group)",
-                    "move table cg{{group}}",
-                ),
-            ],
-        ),
+            panels.timeseries_count(
+                "Fetch Meta Unhits",
+                "",
+                [
+                    panels.target(
+                        f"{metric('state_store_iter_fetch_meta_cache_unhits')}",
+                        "",
+                    ),
+                ],
+            ),
+        ])
+    ]
 
-        panels.timeseries_count(
-            "State Table Count",
-            "The number of state_tables in each CG",
-            [
-                panels.target(
-                    f"sum(irate({table_metric('storage_state_table_count')}[$__rate_interval])) by (group)",
-                    "state table cg{{group}}",
-                ),
-            ],
-        ),
+def section_hummock_write(outer_panels):
+    panels = outer_panels.sub_panel()
+    return [
+        outer_panels.row_collapsed(
+            "Hummock (Write)",
+        [
+            panels.timeseries_bytes(
+                "Uploader Memory Size",
+                "This metric shows the real memory usage of uploader.",
+                [
+                    panels.target(
+                        f"sum({metric('uploading_memory_size')}) by (job,instance)",
+                        "uploading memory - {{job}} @ {{instance}}",
+                    ),
+                    panels.target(
+                        f"sum({metric('state_store_uploader_uploading_task_size')}) by (job,instance)",
+                        "uploading task size - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Build and Sync Sstable Duration",
+                "Histogram of time spent on compacting shared buffer to remote storage.",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({metric('state_store_sync_duration_bucket')}[$__rate_interval])) by (le, job, instance))",
+                            f"p{legend}" + " - {{job}} @ {{instance}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance) (rate({metric('state_store_sync_duration_sum')}[$__rate_interval])) / sum by(le, job, instance) (rate({metric('state_store_sync_duration_count')}[$__rate_interval]))",
+                        "avg - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes(
+                "Materialized View Write Size",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f'sum(histogram_quantile({quantile}, sum(rate({metric("state_store_write_batch_size_bucket")}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group({metric("table_info")}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)',
+                            f"write p{legend} - materialized view {{{{materialized_view_id}}}}"
+                        ),
+                        [50, 99, "max"],
+                    ),
+                ],
+            ),
+            panels.timeseries_count(
+                "Uploader - Tasks Count",
+                "",
+                [
+                    panels.target(
+                        f"sum(irate({table_metric('state_store_merge_imm_task_counts')}[$__rate_interval])) by (job,instance,table_id)",
+                        "merge imm tasks - {{table_id}} @ {{instance}} ",
+                    ),
+                    panels.target(
+                        f"sum(irate({metric('state_store_spill_task_counts')}[$__rate_interval])) by (job,instance,uploader_stage)",
+                        "Uploader spill tasks - {{uploader_stage}} @ {{instance}} ",
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes(
+                "Uploader - Task Size",
+                "",
+                [
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_merge_imm_memory_sz')}[$__rate_interval])) by (job,instance,table_id)",
+                        "Merging tasks memory size - {{table_id}} @ {{instance}} ",
+                    ),
+                    panels.target(
+                        f"sum(rate({metric('state_store_spill_task_size')}[$__rate_interval])) by (job,instance,uploader_stage)",
+                        "Uploading tasks size - {{uploader_stage}} @ {{instance}} ",
+                    ),
+                ],
+            ),
 
-        panels.timeseries_count(
-            "Branched SST Count",
-            "The number of branched_sst in each CG",
-            [
-                panels.target(
-                    f"sum(irate({table_metric('storage_branched_sst_count')}[$__rate_interval])) by (group)",
-                    "branched sst cg{{group}}",
-                ),
-            ],
-        ),
+            panels.timeseries_ops(
+                "Write Ops",
+                "",
+                [
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_write_batch_duration_count')}[$__rate_interval])) by (job,instance,table_id)",
+                        "write batch - {{table_id}} @ {{job}} @ {{instance}} ",
+                    ),
+                    panels.target(
+                        f"sum(rate({metric('state_store_sync_duration_count')}[$__rate_interval])) by (job,instance)",
+                        "l0 - {{job}} @ {{instance}} ",
+                    ),
+                ],
+            ),
+            panels.timeseries_latency(
+                "Write Duration",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_write_batch_duration_bucket')}[$__rate_interval])) by (le, job, instance, table_id))",
+                            f"write to shared_buffer p{legend}" +
+                            " - {{table_id}} @ {{job}} @ {{instance}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance, table_id)(rate({table_metric('state_store_write_batch_duration_sum')}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate({table_metric('state_store_write_batch_duration_count')}[$__rate_interval]))",
+                        "write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}",
+                    ),
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({metric('state_store_write_shared_buffer_sync_time_bucket')}[$__rate_interval])) by (le, job, instance))",
+                            f"write to object_store p{legend}" +
+                            " - {{job}} @ {{instance}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance)(rate({metric('state_store_write_shared_buffer_sync_time_sum')}[$__rate_interval]))  / sum by(le, job, instance)(rate({metric('state_store_write_shared_buffer_sync_time_count')}[$__rate_interval]))",
+                        "write to object_store - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_ops(
+                "Write Item Count",
+                "",
+                [
+                    panels.target(
+                        f"sum(irate({table_metric('state_store_write_batch_tuple_counts')}[$__rate_interval])) by (job,instance,table_id)",
+                        "write_batch_kv_pair_count - {{table_id}} @ {{instance}} ",
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes_per_sec(
+                "Write Throughput",
+                "",
+                [
+                    panels.target(
+                        f"sum(rate({table_metric('state_store_write_batch_size_sum')}[$__rate_interval]))by(job,instance,table_id) / sum(rate({table_metric('state_store_write_batch_size_count')}[$__rate_interval]))by(job,instance,table_id)",
+                        "shared_buffer - {{table_id}} @ {{job}} @ {{instance}}",
+                    ),
+                    panels.target(
+                        f"sum(rate({metric('compactor_shared_buffer_to_sstable_size_sum')}[$__rate_interval]))by(job,instance) / sum(rate({metric('compactor_shared_buffer_to_sstable_size_count')}[$__rate_interval]))by(job,instance)",
+                        "sync - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes(
+                "Mem Table Size (Max)",
+                "This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.",
+                [
+                    panels.target(
+                        f"histogram_quantile(1.0, sum(rate({metric('state_store_write_batch_size_bucket')}[$__rate_interval])) by (le, table_id, job, instance))",
+                        "pmax - {{table_id}} @ {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+            panels.timeseries_bytes(
+                "Checkpoint Sync Size",
+                "",
+                [
+                    *quantile(
+                        lambda quantile, legend: panels.target(
+                            f"histogram_quantile({quantile}, sum(rate({metric('state_store_sync_size_bucket')}[$__rate_interval])) by (le, job, instance))",
+                            f"p{legend}" + " - {{job}} @ {{instance}}",
+                        ),
+                        [50, 99, "max"],
+                    ),
+                    panels.target(
+                        f"sum by(le, job, instance) (rate({metric('state_store_sync_size_sum')}[$__rate_interval])) / sum by(le, job, instance) (rate({metric('state_store_sync_size_count')}[$__rate_interval]))",
+                        "avg - {{job}} @ {{instance}}",
+                    ),
+                ],
+            ),
+        ])
     ]
 
 
@@ -2174,34 +2199,22 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('data_foyer_storage_latency_count')}[$__rate_interval])) by (op, extra, instance)",
-                            "data file cache {{op}} {{extra}} @ {{instance}}",
-                        ),
-                        panels.target(
-                            f"sum(rate({metric('meta_foyer_storage_latency_count')}[$__rate_interval])) by (op, extra, instance)",
-                            "meta cache {{op}} {{extra}} @ {{instance}}",
+                            f"sum(rate({metric('foyer_storage_op_duration_count')}[$__rate_interval])) by (foyer, op, extra, instance)",
+                            "{{foyer}} file cache {{op}} {{extra}} @ {{instance}}",
                         ),
                     ],
                 ),
                 panels.timeseries_latency(
-                    "Latency",
+                    "Duration",
                     "",
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('data_foyer_storage_latency_bucket')}[$__rate_interval])) by (le, op, extra, instance))",
-                                f"p{legend} - data file cache" +
-                                " - {{op}} {{extra}} @ {{instance}}",
-                            ),
-                            [50, 90, 99, "max"],
-                        ),
-                        *quantile(
-                            lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('meta_foyer_storage_latency_bucket')}[$__rate_interval])) by (le, op, extra, instance))",
-                                f"p{legend} - meta file cache" +
-                                " - {{op}} {{extra}} @ {{instance}}",
+                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_op_duration_bucket')}[$__rate_interval])) by (le, foyer, op, extra, instance))",
+                                f"p{legend}" +
+                                " - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}",
                             ),
-                            [50, 90, 99, "max"],
+                            [50, 99, "max"],
                         ),
                     ],
                 ),
@@ -2210,12 +2223,8 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('data_foyer_storage_bytes')}[$__rate_interval])) by (op, extra, instance)",
-                            "data file cache - {{op}} {{extra}} @ {{instance}}",
-                        ),
-                        panels.target(
-                            f"sum(rate({metric('meta_foyer_storage_bytes')}[$__rate_interval])) by (op, extra, instance)",
-                            "meta file cache - {{op}} {{extra}} @ {{instance}}",
+                            f"sum(rate({metric('foyer_storage_op_bytes')}[$__rate_interval])) by (foyer, op, extra, instance)",
+                            "{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}",
                         ),
                     ],
                 ),
@@ -2224,10 +2233,7 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"{metric('data_foyer_storage_size')}", "size @ {{instance}}"
-                        ),
-                            panels.target(
-                            f"{metric('meta_foyer_storage_size')}", "size @ {{instance}}"
+                            f"sum({metric('foyer_storage_total_bytes')}) by (foyer, instance)", "{{foyer}} size @ {{instance}}"
                         ),
                     ],
                 ),
@@ -2236,22 +2242,58 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('data_foyer_storage_latency_count', file_cache_hit_filter)}[$__rate_interval])) by (instance) / (sum(rate({metric('data_foyer_storage_latency_count', file_cache_hit_filter)}[$__rate_interval])) by (instance) + sum(rate({metric('data_foyer_storage_latency_count', file_cache_miss_filter)}[$__rate_interval])) by (instance))",
-                            "data file cache hit ratio @ {{instance}}",
+                            f"sum(rate({metric('foyer_storage_op_duration_count', file_cache_hit_filter)}[$__rate_interval])) by (foyer, instance) / (sum(rate({metric('foyer_storage_op_duration_count', file_cache_hit_filter)}[$__rate_interval])) by (foyer, instance) + sum(rate({metric('foyer_storage_op_duration_count', file_cache_miss_filter)}[$__rate_interval])) by (foyer, instance))",
+                            "{{foyer}} file cache hit ratio @ {{instance}}",
                         ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "Refill Queue Length",
+                    "",
+                    [
                         panels.target(
-                            f"sum(rate({metric('meta_foyer_storage_latency_count', file_cache_hit_filter)}[$__rate_interval])) by (instance) / (sum(rate({metric('meta_foyer_storage_latency_count', file_cache_hit_filter)}[$__rate_interval])) by (instance) + sum(rate({metric('meta_foyer_storage_latency_count', file_cache_miss_filter)}[$__rate_interval])) by (instance))",
-                            "meta file cache hit ratio @ {{instance}}",
+                            f"sum(refill_queue_total) by (instance)",
+                            "refill queue length @ {{instance}}",
                         ),
                     ],
                 ),
                 panels.timeseries_ops(
-                    "Refill",
+                    "Refill Ops",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('compute_refill_data_file_cache_count')}[$__rate_interval])) by (extra, instance)",
-                            "refill data file cache - {{extra}} @ {{instance}}",
+                            f"sum(rate({metric('data_refill_duration_count')}[$__rate_interval])) by (op, instance)",
+                            "data file cache refill - {{op}} @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('data_refill_filtered_total')}[$__rate_interval])) by (instance)",
+                            "data file cache refill - filtered @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('meta_refill_duration_count')}[$__rate_interval])) by (op, instance)",
+                            "meta file cache refill - {{op}} @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_latency(
+                    "Refill Latency",
+                    "",
+                    [
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('data_refill_duration_bucket')}[$__rate_interval])) by (le, op, instance))",
+                                f"p{legend} - " +
+                                "data file cache refill - {{op}} @ {{instance}}",
+                            ),
+                            [50, 99, "max"],
+                        ),
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('meta_refill_duration_bucket')}[$__rate_interval])) by (le, instance))",
+                                f"p{legend} - " +
+                                "meta cache refill @ {{instance}}",
+                            ),
+                            [50, 99, "max"],
                         ),
                     ],
                 ),
@@ -2280,7 +2322,7 @@ def section_hummock_manager(outer_panels):
                                 f"Lock Time p{legend}" +
                                 " - {{lock_type}} @ {{lock_name}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
                         ),
                     ],
                 ),
@@ -2294,7 +2336,7 @@ def section_hummock_manager(outer_panels):
                                 f"Real Process Time p{legend}" +
                                 " - {{method}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
                         ),
                     ],
                 ),
@@ -2455,7 +2497,7 @@ def section_hummock_manager(outer_panels):
                                 f"meta consumed latency p{legend}" +
                                 " - {{job}} @ {{instance}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
                         ),
 
                         *quantile(
@@ -2464,7 +2506,7 @@ def section_hummock_manager(outer_panels):
                                 f"meta iteration latency p{legend}" +
                                 " - {{job}} @ {{instance}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
                         ),
 
                         *quantile(
@@ -2473,7 +2515,7 @@ def section_hummock_manager(outer_panels):
                                 f"compactor consumed latency p{legend}" +
                                 " - {{job}} @ {{instance}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
                         ),
 
                         *quantile(
@@ -2482,7 +2524,39 @@ def section_hummock_manager(outer_panels):
                                 f"compactor iteration latency p{legend}" +
                                 " - {{job}} @ {{instance}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
+                        ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "Move State Table Count",
+                    "The times of move_state_table occurs",
+                    [
+                        panels.target(
+                            f"sum({table_metric('storage_move_state_table_count')}) by (group)",
+                            "move table cg{{group}}",
+                        ),
+                    ],
+                ),
+
+                panels.timeseries_count(
+                    "State Table Count",
+                    "The number of state_tables in each CG",
+                    [
+                        panels.target(
+                            f"sum(irate({table_metric('storage_state_table_count')}[$__rate_interval])) by (group)",
+                            "state table cg{{group}}",
+                        ),
+                    ],
+                ),
+
+                panels.timeseries_count(
+                    "Branched SST Count",
+                    "The number of branched_sst in each CG",
+                    [
+                        panels.target(
+                            f"sum(irate({table_metric('storage_branched_sst_count')}[$__rate_interval])) by (group)",
+                            "branched sst cg{{group}}",
                         ),
                     ],
                 ),
@@ -2517,7 +2591,7 @@ def section_backup_manager(outer_panels):
                                 f"Job Process Time p{legend}" +
                                 " - {{state}}",
                             ),
-                            [50, 99, 999, "max"],
+                            [50, 99, "max"],
                         ),
                     ],
                 ),
@@ -3174,6 +3248,128 @@ def section_connector_node(outer_panels):
         )
     ]
 
+def section_network_connection(outer_panels):
+    panels = outer_panels.sub_panel()
+    s3_filter = 'connection_type="S3"'
+    grpc_filter = 'connection_type=~"grpc.*"'
+    return [
+        outer_panels.row_collapsed(
+            "Network connection",
+            [
+                panels.timeseries_bytesps(
+                    "Network throughput",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('connection_read_rate')}[$__rate_interval])) by (job, instance) / (1024*1024)",
+                            "{{job}} read @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_write_rate')}[$__rate_interval])) by (job, instance) / (1024*1024)",
+                            "{{job}} write @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_bytesps(
+                    "S3 throughput",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('connection_read_rate', filter=s3_filter)}[$__rate_interval])) by (job, instance) / (1024*1024)",
+                            "{{job}} read @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_write_rate', filter=s3_filter)}[$__rate_interval])) by (job, instance) / (1024*1024)",
+                            "{{job}} write @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_bytesps(
+                    "gRPC throughput",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('connection_read_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)",
+                            "{{job}} {{connection_type}} read @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_write_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)",
+                            "{{job}} {{connection_type}} write @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_read_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance) / (1024*1024)",
+                            "{{job}} total read @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_write_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance) / (1024*1024)",
+                            "{{job}} total write @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "IO error rate",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(irate({metric('connection_io_err_rate', filter=s3_filter)}[$__rate_interval])) by (job, instance, op_type, error_kind)",
+                            "{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_io_err_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance, op_type, error_kind)",
+                            "{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('connection_io_err_rate')}[$__rate_interval])) by (job, instance, op_type, error_kind)",
+                            "{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "Existing connection count",
+                    "",
+                    [
+                        panels.target(
+                            f"sum({metric('connection_count', filter=s3_filter)}) by (job, instance)",
+                            "{{job}} S3 @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum({metric('connection_count', filter=grpc_filter)}) by (job, instance, connection_type)",
+                            "{{job}} {{connection_type}} @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "Create new connection rate",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(irate({metric('connection_create_rate', filter=s3_filter)}[$__rate_interval])) by (job, instance)",
+                            "{{job}} S3 @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(irate({metric('connection_create_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance, connection_type)",
+                            "{{job}} {{connection_type}} @ {{instance}}",
+                        ),
+                    ],
+                ),
+                panels.timeseries_count(
+                    "Create new connection err rate",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(irate({metric('connection_err_rate', filter=s3_filter)}[$__rate_interval])) by (job, instance)",
+                            "{{job}} S3 @ {{instance}}",
+                        ),
+                        panels.target(
+                            f"sum(irate({metric('connection_err_rate', filter=grpc_filter)}[$__rate_interval])) by (job, instance, connection_type)",
+                            "{{job}} {{connection_type}} @ {{instance}}",
+                        ),
+                    ],
+                ),
+            ],
+        )
+    ]
+
 
 templating_list = []
 if dynamic_source_enabled:
@@ -3350,7 +3546,8 @@ def section_connector_node(outer_panels):
         *section_streaming_exchange(panels),
         *section_streaming_errors(panels),
         *section_batch(panels),
-        *section_hummock(panels),
+        *section_hummock_read(panels),
+        *section_hummock_write(panels),
         *section_compaction(panels),
         *section_object_storage(panels),
         *section_hummock_tiered_cache(panels),
@@ -3365,5 +3562,6 @@ def section_connector_node(outer_panels):
         *section_memory_manager(panels),
         *section_connector_node(panels),
         *section_kafka_native_metrics(panels),
+        *section_network_connection(panels)
     ],
 ).auto_panel_ids()
diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json
index 6e8d945a4ba11..b57021d2cef98 100644
--- a/grafana/risingwave-dev-dashboard.json
+++ b/grafana/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":27},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":35},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":44},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":52},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":60},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","query":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":60},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":68},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":68},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows output by each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":76},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_name}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":76},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":84},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":84},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":92},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":92},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":100},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":100},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":108},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":108},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":116},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}->{{executor_identity}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Backpressure","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}->{{upstream_fragment_id}}","metric":"","query":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, actor_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,actor_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg {{actor_id}}.{{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, actor_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,actor_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Entries","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the size of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Estimated Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - actor_id {{actor_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":117},"height":null,"hideTimeOverride":false,"id":61,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":118},"height":null,"hideTimeOverride":false,"id":77,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":119},"height":null,"hideTimeOverride":false,"id":80,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":120},"height":null,"hideTimeOverride":false,"id":83,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":121},"height":null,"hideTimeOverride":false,"id":88,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":122},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":122},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(sstable_preload_io_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"preload iops","metric":"","query":"sum(rate(sstable_preload_io_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) ","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":130},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"File Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":130},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_range_reverse_scan_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"backward scan - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_range_reverse_scan_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":138},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":138},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":146},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.999, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.999, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":146},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":154},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p90 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.9, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.9, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.9, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":154},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.9, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p90 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.9, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":162},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":162},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":170},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":170},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_may_exist_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_may_exist_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_may_exist_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_may_exist_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_may_exist_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - MayExist","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":178},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter miss count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Bloom Filter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":178},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":186},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) / (sum(rate(file_cache_latency_count{op='get',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss rate @ {{instance}}","metric":"","query":"(sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) / (sum(rate(file_cache_latency_count{op='get',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":186},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter miss rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom-Filter Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":194},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Request Bloom-Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":194},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p90 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":202},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":202},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":210},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":210},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":218},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":218},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":226},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size (Max)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":226},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block Cache 3. Uploader.This metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":234},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":234},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":242},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":242},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":250},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":250},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":258},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":258},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":266},"height":null,"hideTimeOverride":false,"id":125,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success & Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that’s another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":267},"height":null,"hideTimeOverride":false,"id":150,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":268},"height":null,"hideTimeOverride":false,"id":159,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(data_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(meta_foyer_storage_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(data_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_foyer_storage_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(data_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(meta_foyer_storage_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"data_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"size @ {{instance}}","metric":"","query":"data_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"size @ {{instance}}","metric":"","query":"meta_foyer_storage_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(data_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(meta_foyer_storage_latency_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compute_refill_data_file_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill data file cache - {{extra}} @ {{instance}}","metric":"","query":"sum(rate(compute_refill_data_file_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":269},"height":null,"hideTimeOverride":false,"id":166,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p999 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.999, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p999 - {{method}}","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p999 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":270},"height":null,"hideTimeOverride":false,"id":183,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p999 - {{state}}","metric":"","query":"histogram_quantile(0.999, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":271},"height":null,"hideTimeOverride":false,"id":186,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":272},"height":null,"hideTimeOverride":false,"id":190,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":273},"height":null,"hideTimeOverride":false,"id":193,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":274},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":275},"height":null,"hideTimeOverride":false,"id":202,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":276},"height":null,"hideTimeOverride":false,"id":210,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":277},"height":null,"hideTimeOverride":false,"id":219,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":278},"height":null,"hideTimeOverride":false,"id":226,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":279},"height":null,"hideTimeOverride":false,"id":229,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time since this client instance was created (milli seconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Client Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":231,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Broker Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Topic Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":233,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Partition Level Metrics","transformations":[],"transparent":false,"type":"row"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"actor_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"rate(partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","query":"(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","query":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows output by each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_name}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*SinkExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(sink_name) (group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, sink_name))) by (sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":33,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}->{{executor_identity}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Backpressure","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Materialzed View Memory Usage","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id, actor_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}->{{upstream_fragment_id}}","metric":"","query":"rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_barrier_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_processing_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Processing Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Row","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}} ","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total lookups {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss when insert {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_insert_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, fragment_id, wait_side, job, instance)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job,instance) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}.{{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entries{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Entries","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_rows{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the size of rows in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}} {{side}}","metric":"","query":"stream_join_cached_estimated_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Estimated Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Lookup miss count counts the number of aggregation key's cache miss per second.Lookup total count counts the number of rows processed per second.By diving these two metrics, one can derive the cache miss rate per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each Key/State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count |table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_keys{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cached_entry_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cached entry count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_over_window_cached_entry_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} - actor {{actor_id}}   {{instance}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":63,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":79,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":86,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":92,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job,instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instanc,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":113,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}} ","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Size (Max)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":125,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_split_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}} ","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":151,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":160,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","query":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(data_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(data_refill_filtered_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data file cache refill - filtered @ {{instance}}","metric":"","query":"sum(rate(data_refill_filtered_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(meta_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(meta_refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - data file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - data file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - data file cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(data_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - meta cache refill @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - meta cache refill @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - meta cache refill @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":169,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":189,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":196,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":199,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":203,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}} ","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}} ","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}} ","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}} ","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}} ","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":216,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":225,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_evicted_watermark_time_diff_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":235,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time since this client instance was created (milli seconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_age{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Client Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Broker Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":238,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Topic Level Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":239,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Partition Level Metrics","transformations":[],"transparent":false,"type":"row"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":240,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/integration_tests/cassandra-and-syclladb-sink/README.md b/integration_tests/cassandra-and-syclladb-sink/README.md
new file mode 100644
index 0000000000000..b022c9ef09cf8
--- /dev/null
+++ b/integration_tests/cassandra-and-syclladb-sink/README.md
@@ -0,0 +1,65 @@
+# Demo: Sinking to Cassandra/Scylladb
+
+In this demo, we want to showcase how RisingWave is able to sink data to Cassandra.
+
+1. Set the compose profile accordingly:
+Demo with Apache Cassandra:
+```
+export COMPOSE_PROFILES=cassandra
+```
+
+Demo with Scylladb
+```
+export COMPOSE_PROFILES=scylladb
+```
+
+2. Launch the cluster:
+
+```sh
+docker-compose up -d
+```
+
+The cluster contains a RisingWave cluster and its necessary dependencies, a datagen that generates the data, a Cassandra for sink.
+
+
+3. Create the Cassandra table via cqlsh:
+
+Login to cqlsh
+```sh
+# cqlsh into cassandra
+docker compose exec cassandra cqlsh
+# cqlsh into scylladb
+docker compose exec scylladb cqlsh
+```
+
+Run the following queries to create keyspace and table.
+```sql
+CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+use demo;
+CREATE table demo_bhv_table(
+    user_id int primary key,
+    target_id text,
+    event_timestamp timestamp,
+);
+```
+
+3. Execute the SQL queries in sequence:
+
+- create_source.sql
+- create_mv.sql
+- create_sink.sql
+
+4. Execute a simple query to check the sink results via csqlsh:
+
+Login to cqlsh
+```sh
+# cqlsh into cassandra
+docker compose exec cassandra cqlsh
+# cqlsh into scylladb
+docker compose exec scylladb cqlsh
+```
+
+Run the following query
+```sql
+select user_id, count(*) from my_keyspace.demo_test group by user_id;
+```
diff --git a/integration_tests/cassandra-and-syclladb-sink/create_mv.sql b/integration_tests/cassandra-and-syclladb-sink/create_mv.sql
new file mode 100644
index 0000000000000..0a803f8a2762d
--- /dev/null
+++ b/integration_tests/cassandra-and-syclladb-sink/create_mv.sql
@@ -0,0 +1,7 @@
+CREATE MATERIALIZED VIEW bhv_mv AS
+SELECT
+    user_id,
+    target_id,
+    event_timestamp
+FROM
+    user_behaviors;
\ No newline at end of file
diff --git a/integration_tests/cassandra-and-syclladb-sink/create_sink.sql b/integration_tests/cassandra-and-syclladb-sink/create_sink.sql
new file mode 100644
index 0000000000000..724e784694c2f
--- /dev/null
+++ b/integration_tests/cassandra-and-syclladb-sink/create_sink.sql
@@ -0,0 +1,11 @@
+CREATE SINK bhv_cassandra_sink
+FROM
+    bhv_mv WITH (
+    connector = 'cassandra',
+    type = 'append-only',
+    force_append_only='true',
+    cassandra.url = 'cassandra:9042',
+    cassandra.keyspace  = 'demo',
+    cassandra.table = 'demo_bhv_table',
+    cassandra.datacenter = 'datacenter1',
+);
\ No newline at end of file
diff --git a/integration_tests/cassandra-and-syclladb-sink/create_source.sql b/integration_tests/cassandra-and-syclladb-sink/create_source.sql
new file mode 100644
index 0000000000000..c28c10f3616da
--- /dev/null
+++ b/integration_tests/cassandra-and-syclladb-sink/create_source.sql
@@ -0,0 +1,18 @@
+CREATE table user_behaviors (
+    user_id int,
+    target_id VARCHAR,
+    target_type VARCHAR,
+    event_timestamp TIMESTAMP,
+    behavior_type VARCHAR,
+    parent_target_type VARCHAR,
+    parent_target_id VARCHAR,
+    PRIMARY KEY(user_id)
+) WITH (
+    connector = 'datagen',
+    fields.user_id.kind = 'sequence',
+    fields.user_id.start = '1',
+    fields.user_id.end = '1000',
+    fields.user_name.kind = 'random',
+    fields.user_name.length = '10',
+    datagen.rows.per.second = '10'
+) FORMAT PLAIN ENCODE JSON;
\ No newline at end of file
diff --git a/integration_tests/cassandra-and-syclladb-sink/docker-compose.yml b/integration_tests/cassandra-and-syclladb-sink/docker-compose.yml
new file mode 100644
index 0000000000000..27b77f850f882
--- /dev/null
+++ b/integration_tests/cassandra-and-syclladb-sink/docker-compose.yml
@@ -0,0 +1,82 @@
+---
+version: "3"
+services:
+  cassandra:
+    image: cassandra:4.0
+    ports:
+      - 9042:9042
+    environment:
+      - CASSANDRA_CLUSTER_NAME=cloudinfra
+    profiles:
+      - cassandra
+  scylladb:
+    image: scylladb/scylla:5.1
+    ports:
+      - 9042:9042
+    environment:
+      - CASSANDRA_CLUSTER_NAME=cloudinfra
+    profiles:
+      - scylladb
+  compactor-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: compactor-0
+  compute-node-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: compute-node-0
+  etcd-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: etcd-0
+  frontend-node-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: frontend-node-0
+  grafana-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: grafana-0
+  meta-node-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: meta-node-0
+  connector-node:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: connector-node
+  minio-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: minio-0
+  prometheus-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: prometheus-0
+  message_queue:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: message_queue
+  datagen:
+    build: ../datagen
+    depends_on: [message_queue]
+    command:
+      - /bin/sh
+      - -c
+      - /datagen --mode clickstream --qps 2 kafka --brokers message_queue:29092
+    restart: always
+    container_name: datagen
+volumes:
+  compute-node-0:
+    external: false
+  etcd-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+  message_queue:
+    external: false
+name: risingwave-compose
diff --git a/integration_tests/feature-store/Dockerfile b/integration_tests/feature-store/Dockerfile
new file mode 100644
index 0000000000000..dc0d02bd79682
--- /dev/null
+++ b/integration_tests/feature-store/Dockerfile
@@ -0,0 +1,59 @@
+FROM rust:1.67 as feature-store-server
+ARG BUILD_ARG
+
+USER root
+
+ENV WORK_DIR /opt/feature-store
+RUN mkdir -p $WORK_DIR
+
+WORKDIR $WORK_DIR
+
+RUN apt update
+RUN apt install -y python3 python3-pip wget ca-certificates
+RUN apt install -y postgresql-client
+
+ADD ./server/model/requirements.txt $WORK_DIR/model-pipreqs.txt
+ADD ./generator/requirements.txt $WORK_DIR/generator-pipreqs.txt
+RUN pip3 install --upgrade pip
+RUN pip3 install -r $WORK_DIR/model-pipreqs.txt
+RUN pip3 install -r $WORK_DIR/generator-pipreqs.txt
+RUN pip3 install risingwave
+
+RUN apt install -y lsof curl openssl libssl-dev pkg-config build-essential
+RUN apt install -y cmake librdkafka-dev
+
+# Install .NET 6.0
+RUN wget https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
+RUN dpkg -i packages-microsoft-prod.deb
+RUN rm packages-microsoft-prod.deb
+RUN apt-get update && apt-get install -y dotnet-sdk-6.0
+RUN apt install -y liblttng-ust0
+
+# `cargo build` included in ./build
+ADD ./server $WORK_DIR/build/server
+ADD ./simulator $WORK_DIR/build/simulator
+RUN cargo build --manifest-path $WORK_DIR/build/server/Cargo.toml --release
+RUN cargo build --manifest-path $WORK_DIR/build/simulator/Cargo.toml --release
+
+RUN cp $WORK_DIR/build/server/target/release/server $WORK_DIR/feature-store-server
+RUN cp $WORK_DIR/build/simulator/target/release/simulator $WORK_DIR/feature-store-simulator
+RUN rm -rf $WORK_DIR/build
+
+ADD ./server/model $WORK_DIR/server/model
+ADD ./server/udf.py $WORK_DIR/udf.py
+ADD ./generator $WORK_DIR/generator
+ADD ./taxi-start.sql $WORK_DIR/taxi-start.sql
+ADD ./mfa-start.sql $WORK_DIR/mfa-start.sql
+RUN mkdir $WORK_DIR/run-sh
+ADD ./run.sh $WORK_DIR/run-sh/
+ADD ./run-mfa.sh $WORK_DIR/run-sh/
+
+RUN if [ "$BUILD_ARG" = "mfa" ]; then \
+        cp $WORK_DIR/run-sh/run-mfa.sh $WORK_DIR/run.sh;\
+    else \
+        cp $WORK_DIR/run-sh/run.sh $WORK_DIR/run.sh;\
+    fi
+
+RUN chmod +x $WORK_DIR/run.sh && rm -rf $WORK_DIR/run-sh
+
+CMD ["sh", "-c", "sleep 10 && ./run.sh"]
\ No newline at end of file
diff --git a/integration_tests/feature-store/README.md b/integration_tests/feature-store/README.md
new file mode 100644
index 0000000000000..425efadf27c68
--- /dev/null
+++ b/integration_tests/feature-store/README.md
@@ -0,0 +1,60 @@
+# Description
+
+Feature store demo.
+
+We use `simulators` to simulate data input.
+
+Then all messages will be sent to the `server` and written in `Kafka` -> `RisingWave`. `RisingWave` will process the data based on pre-defined operations.
+
+We also utilize the `simulator` to simulate user queries to our `feature`. The `server` will receive requests -> query data -> and return results.
+
+If we intend to modify our business logic, we simply need to update the materialized view within our `RisingWave` by using SQL statements.
+
+# Nyc taxi feature store
+#### Case Description
+
+The case in this chapter is a New York taxi fare prediction. We need to predict the taxi fare based on the starting and ending points of the trip.
+
+We use the starting and ending points as primary keys, extract and transform corresponding features, and save them in `RisingWave`. These features are updated based on user behavior.
+
+When a user needs to make a prediction using these features, they can retrieve all the features for training.
+
+When a user needs to make a prediction using these features, they can provide their starting and ending points, query the corresponding features in `RisingWave`, and inject them into a machine learning model for real-time fare prediction.
+
+#### Installation
+
+1. Build docker. Kafka RisingWave and Feature Store.
+
+```docker compose up --build```
+
+
+The Feature Store system performs several tasks in sequence:
+
+- Writes simulated offline data into `Kafka`→`RisingWave`, extracting behavior and feature tables.
+
+- Joins the behavior and feature tables to obtain corresponding offline training data and conducts model training.
+
+- Writes simulated online feature data into `Kafka`→`RisingWave`.
+
+- Uses `do_location_id` (destination location) and `pu_location_id` (pickup location) to query the latest online features in RisingWave and utilizes these online features along with the trained model for predictions.
+
+2. Then we can get the simulation results for Feature store in `.log`.
+
+```cat .log/simulator_log```
+
+# Account change feature store
+#### Case Description
+
+This chapter is a simple demo of feature extraction in `RisingWave`, primarily showcasing the real-time feature aggregation and updating capabilities of `RisingWave`.
+
+In this case, we need to calculate the frequency and count of user account changes over a period of time. We mainly use SQL aggregation functions and UDFs (User-Defined Functions) to achieve this.
+
+#### Installation
+
+1. Build docker. Kafka RisingWave and Feature Store.
+
+```docker compose build --build-arg BUILD_ARG=mfa```
+
+2. Then we can get the simulation results for Feature store in `.log`.
+
+```cat .log/simulator_log```
\ No newline at end of file
diff --git a/integration_tests/feature-store/docker-compose.yml b/integration_tests/feature-store/docker-compose.yml
new file mode 100644
index 0000000000000..d212a80369a38
--- /dev/null
+++ b/integration_tests/feature-store/docker-compose.yml
@@ -0,0 +1,119 @@
+---
+version: "3"
+services:
+  kafka:
+    image: confluentinc/cp-kafka:7.1.0
+    platform: linux/amd64
+    hostname: kafka
+    container_name: kafka
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR
+    depends_on:
+      [ zookeeper ]
+    healthcheck:
+      test: [ "CMD-SHELL", "kafka-topics --bootstrap-server kafka:9092 --list" ]
+      interval: 5s
+      timeout: 10s
+      retries: 5
+
+  init-kafka:
+    image: confluentinc/cp-kafka:7.1.0
+    depends_on:
+      - kafka
+    entrypoint: [ '/bin/sh', '-c' ]
+    command: |
+      "
+      # blocks until kafka is reachable
+      kafka-topics --bootstrap-server kafka:9092 --list
+      echo -e 'Creating kafka topics'
+      kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic taxi --replication-factor 1 --partitions 1
+      echo -e 'Creating kafka topics'
+      kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic mfa --replication-factor 1 --partitions 1
+
+      echo -e 'Successfully created the following topics:'
+      kafka-topics --bootstrap-server kafka:9092 --list
+      "
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.1.0
+    platform: linux/amd64
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+  compactor-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: compactor-0
+  compute-node-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: compute-node-0
+    volumes:
+      - "./server/udf.py:/udf.py"
+      - "./mfa-start.sql:/mfa-start.sql"
+      - "./mfa-mock.sql:/mfa-mock.sql"
+  feature-store:
+    image: rust:1.67
+    build:
+      context: .
+      target: feature-store-server
+    depends_on:
+      [kafka,meta-node-0,frontend-node-0]
+    volumes:
+      - ".log:/opt/feature-store/.log"
+  etcd-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: etcd-0
+  frontend-node-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: frontend-node-0
+  grafana-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: grafana-0
+  meta-node-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: meta-node-0
+    ports:
+      - "8815:8815"
+    depends_on:
+      [kafka]
+  minio-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: minio-0
+  prometheus-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: prometheus-0
+  connector-node:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: connector-node
+volumes:
+  etcd-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+name: risingwave-compose
+
diff --git a/integration_tests/feature-store/generator/__init__.py b/integration_tests/feature-store/generator/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/integration_tests/feature-store/generator/__main__.py b/integration_tests/feature-store/generator/__main__.py
new file mode 100644
index 0000000000000..52bd2e4ec457c
--- /dev/null
+++ b/integration_tests/feature-store/generator/__main__.py
@@ -0,0 +1,30 @@
+import json
+import click
+import user
+    # This is a package in preview.
+from azureml.opendatasets import NycTlcGreen
+
+from datetime import datetime
+from dateutil import parser
+
+@click.command()
+@click.option('--types', default='taxi', help='taxi or mfa')
+@click.option('--num-users', default=10, help='Number of users to be generated')
+@click.option('--dump-users', default='./users.json', help="The location to dump the `user` json file")
+def generate(num_users, dump_users,types):
+    if types == 'taxi':
+        get_data_from_azure()
+    else:
+        users = [user.new_user() for _ in range(num_users)]
+        json.dump(users, open(dump_users, 'w'), indent=2)
+
+def get_data_from_azure():
+    end_date = parser.parse('2018-05-02')
+    start_date = parser.parse('2018-05-01')
+    print("Getting data from {} to {}".format(start_date, end_date))
+    nyc_tlc = NycTlcGreen(start_date=start_date, end_date=end_date)
+    nyc_tlc_df = nyc_tlc.to_pandas_dataframe()
+    nyc_tlc_df.iloc[0:10000].to_csv('/opt/feature-store/parquet_data.csv')
+
+if __name__ == '__main__':
+    generate()
diff --git a/integration_tests/feature-store/generator/preference.py b/integration_tests/feature-store/generator/preference.py
new file mode 100644
index 0000000000000..79d63eaf14998
--- /dev/null
+++ b/integration_tests/feature-store/generator/preference.py
@@ -0,0 +1,29 @@
+from functools import partial, reduce
+import numpy as np
+
+
+def _chain(*funcs):
+    def inner():
+        res = None
+        for f in funcs:
+            if not res:
+                res = f()
+            else:
+                res = f(res)
+        return res
+    return inner
+
+
+
+class UserProperties:
+    generators = {
+        "activeness": _chain(
+            partial(np.random.lognormal, mean=10, sigma=5),
+            partial(np.clip, a_min=1, a_max=50)
+        ),
+        "address_lat": partial(np.random.uniform, low=-180, high=180),
+        "address_long": partial(np.random.uniform, low=-180, high=180),
+        "age_approx": partial(np.random.randint, low=18, high=100),
+        "gender": partial(np.random.choice, [0, 1]),
+        "occupation": partial(np.random.randint, low=0, high=25),
+    }
diff --git a/integration_tests/feature-store/generator/requirements.txt b/integration_tests/feature-store/generator/requirements.txt
new file mode 100644
index 0000000000000..4cafe5dc9bea5
--- /dev/null
+++ b/integration_tests/feature-store/generator/requirements.txt
@@ -0,0 +1,2 @@
+click==8.1.3
+azureml-opendatasets==1.52.0
diff --git a/integration_tests/feature-store/generator/user.py b/integration_tests/feature-store/generator/user.py
new file mode 100644
index 0000000000000..0a478c796541c
--- /dev/null
+++ b/integration_tests/feature-store/generator/user.py
@@ -0,0 +1,20 @@
+import uuid
+from collections.abc import Iterable
+from pprint import pprint
+import numpy as np
+from preference import UserProperties
+
+
+def new_user():
+    id = str(np.random.randint(1, 1000_000_000))
+
+    activeness = np.exp(np.random.lognormal(mean=1))
+    distrib = dict(userid=id, activeness=activeness)
+
+    for tag, gen in UserProperties.generators.items():
+        distrib[tag] = float(gen())
+    return distrib
+
+
+if __name__ == "__main__":
+    pprint(new_user(), indent=2)
diff --git a/integration_tests/feature-store/mfa-mock.sql b/integration_tests/feature-store/mfa-mock.sql
new file mode 100644
index 0000000000000..25955f763b8b4
--- /dev/null
+++ b/integration_tests/feature-store/mfa-mock.sql
@@ -0,0 +1,36 @@
+create table actionhistory(
+  userid varchar,
+  eventype varchar, -- mfa+,mfa-,other
+  timestamp timestamp,
+  changenum int,
+);
+
+insert into actionhistory values
+    ('user1', 'mfa-', '2016-02-01 00:00:01',50),
+    ('user1', 'other', '2016-02-01 00:00:03',50),
+    ('user1', 'mfa-', '2016-02-01 00:00:05',100),
+    ('user1', 'mfa+', '2016-02-01 00:01:07',50),
+    ('user1', 'mfa+', '2016-02-01 00:01:09',20),
+    ('user1', 'other', '2016-02-01 00:01:11',50),
+    ('user2', 'other', '2016-02-01 00:00:13',50),
+    ('user2', 'mfa+', '2016-02-01 00:00:15',10),
+    ('user2', 'mfa+', '2016-02-01 00:00:17',10),
+    ('user2', 'mfa-', '2016-02-01 00:01:19',50),
+    ('user2', 'mfa+', '2016-02-01 00:01:21',10),
+    ('user2', 'mfa-', '2016-02-01 00:01:23',20);
+
+create materialized view user_action_mfa as select userid, timestamp,changenum,eventype from actionhistory where eventype in ('mfa-','mfa+');
+
+create materialized view user_mfa_change_count as
+      select userid , count(*) as count, window_start
+      from(
+        select * from tumble(user_action_mfa , timestamp , INTERVAL '30 minutes')
+      ) group by userid,window_start;
+
+create function udf_sum(int,varchar) returns int as udf_sum using link 'http://localhost:8815';
+
+create materialized view user_mfa_change_num as
+      select userid , sum(udf_sum(changenum,eventype)) as sum, window_start
+      from(
+        select * from tumble(user_action_mfa , timestamp , INTERVAL '30 minutes')
+      ) group by userid,window_start;
\ No newline at end of file
diff --git a/integration_tests/feature-store/mfa-start.sql b/integration_tests/feature-store/mfa-start.sql
new file mode 100644
index 0000000000000..48308f403898d
--- /dev/null
+++ b/integration_tests/feature-store/mfa-start.sql
@@ -0,0 +1,27 @@
+create source if not exists actionhistory (
+    userid varchar,
+    eventype varchar, -- mfa+,mfa-,other
+    timestamp timestamp,
+    changenum int,
+) with (
+    connector = 'kafka',
+    topic = 'mfa',
+    properties.bootstrap.server = 'kafka:9092',
+)
+FORMAT PLAIN ENCODE JSON;
+
+create materialized view user_action_mfa as select * from actionhistory where eventype in ('mfa+','mfa-');
+
+create materialized view user_mfa_change_count as
+      select userid , count(*) as count, window_start
+      from(
+        select * from tumble(user_action_mfa , timestamp , INTERVAL '30 minutes')
+      ) group by userid,window_start;
+
+create function udf_sum(int,varchar) returns int as udf_sum using link 'http://feature-store:8815';
+
+create materialized view user_mfa_change_sum as
+      select userid , sum(udf_sum(changenum,eventype)) as udf_sum, window_start
+      from(
+        select * from tumble(user_action_mfa , timestamp , INTERVAL '30 minutes')
+      ) group by userid,window_start;
\ No newline at end of file
diff --git a/integration_tests/feature-store/run-local.sh b/integration_tests/feature-store/run-local.sh
new file mode 100755
index 0000000000000..fbe7ea0a653ee
--- /dev/null
+++ b/integration_tests/feature-store/run-local.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+set -ex
+
+echo "starting zookeeper"
+/usr/local/kafka/kafka_2.12-3.5.0/bin/zookeeper-server-start.sh /usr/local/kafka/kafka_2.12-3.5.0/config/zookeeper.properties > /dev/null 2>&1 &
+sleep 5
+lsof -i:2181  > /dev/null || {
+  echo "failed to start zookeeper"
+  exit 1
+}
+
+echo "zookeeper started. starting kafka"
+/usr/local/kafka/kafka_2.12-3.5.0/bin/kafka-server-start.sh /usr/local/kafka/kafka_2.12-3.5.0/config/server.properties > /dev/null 2>&1 &
+sleep 5
+lsof -i:9092 > /dev/null || {
+  echo "kafka start failed"
+  exit 1
+}
+
+/usr/local/kafka/kafka_2.12-3.5.0/bin/kafka-topics.sh --create --topic taxi --partitions 1 --replication-factor 1 --bootstrap-server=127.0.0.1:9092 \
+  --if-not-exists  > /dev/null 2>&1 || {
+  echo "kafka topic creation failed"
+  exit 1
+}
+sleep 3
+
+# use it in mfa
+# pip install risingwave
+# python3 server/udf.py > /dev/null 2>&1 &
+# lsof -i:8815  > /dev/null || {
+#   echo "failed to start udf"
+#   exit 1
+# }
+# sleep 5
+
+psql -U root -h 127.0.0.1 -p 4566 -d dev -a -f taxi-start.sql || {
+  echo "failed to initialize db for taxi"
+  exit 1
+}
+sleep 2
+
+# use it in mfa
+# export GENERATOR_PATH=generator
+# python3 generator --num-users=15 \
+#   --dump-users="$GENERATOR_PATH/users.json"
+
+pip3 install -r server/model/requirements.txt
\ No newline at end of file
diff --git a/integration_tests/feature-store/run-mfa.sh b/integration_tests/feature-store/run-mfa.sh
new file mode 100644
index 0000000000000..3c9ab9d1f1b6c
--- /dev/null
+++ b/integration_tests/feature-store/run-mfa.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+set -ex
+
+python3 /opt/feature-store/udf.py > /dev/null 2>&1&
+sleep 5
+lsof -i:8815  > /dev/null || {
+  echo "failed to start udf"
+  exit 1
+}
+sleep 5
+
+psql -U root -h frontend-node-0 -p 4566 -d dev -a -f mfa-start.sql || {
+  echo "failed to initialize db for mfa"
+  exit 1
+}
+sleep 2
+
+export GENERATOR_PATH=generator
+python3 generator --types user --num-users=15 \
+  --dump-users="$GENERATOR_PATH/users.json"
+sleep 2
+
+./feature-store-server --output-topics mfa > /opt/feature-store/.log/server_log &
+RECOMMENDER_PID=$!
+sleep 2
+./feature-store-simulator --types mfa > /opt/feature-store/.log/simulator_log &
+SIMULATOR_PID=$!
+
+trap 'kill $SIMULATOR_PID; kill $RECOMMENDER_PID; kill $MODEL_PID' SIGINT
+wait $SIMULATOR_PID
+echo "Simulator finished"
+wait $RECOMMENDER_PID
+echo "Recommender finished"
+wait $MODEL_PID
+echo "Model finished"
\ No newline at end of file
diff --git a/integration_tests/feature-store/run.sh b/integration_tests/feature-store/run.sh
new file mode 100644
index 0000000000000..cea9ee75b3905
--- /dev/null
+++ b/integration_tests/feature-store/run.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -ex
+
+psql -U root -h frontend-node-0 -p 4566 -d dev -a -f taxi-start.sql || {
+  echo "failed to initialize db for taxi"
+  exit 1
+}
+sleep 2
+
+python3 generator
+sleep 2
+
+python3 server/model  > /opt/feature-store/.log/model_log &
+MODEL_PID=$!
+./feature-store-server > /opt/feature-store/.log/server_log &
+RECOMMENDER_PID=$!
+sleep 2
+./feature-store-simulator > /opt/feature-store/.log/simulator_log &
+SIMULATOR_PID=$!
+
+trap 'kill $SIMULATOR_PID; kill $RECOMMENDER_PID; kill $MODEL_PID' SIGINT
+wait $SIMULATOR_PID
+echo "Simulator finished"
+wait $RECOMMENDER_PID
+echo "Recommender finished"
+wait $MODEL_PID
+echo "Model finished"
\ No newline at end of file
diff --git a/integration_tests/feature-store/server/Cargo.lock b/integration_tests/feature-store/server/Cargo.lock
new file mode 100644
index 0000000000000..a678176645772
--- /dev/null
+++ b/integration_tests/feature-store/server/Cargo.lock
@@ -0,0 +1,2442 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "addr2line"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
+name = "aho-corasick"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+
+[[package]]
+name = "async-stream"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "atoi"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "axum"
+version = "0.5.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bitflags 1.3.2",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "sync_wrapper",
+ "tokio",
+ "tower",
+ "tower-http",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "mime",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "backtrace"
+version = "0.3.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "base64"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "bytes"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+
+[[package]]
+name = "cc"
+version = "1.0.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags 1.3.2",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "cmake"
+version = "0.1.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crc"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403"
+
+[[package]]
+name = "crossbeam-queue"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "dirs"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
+[[package]]
+name = "dotenv"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
+
+[[package]]
+name = "either"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "errno"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "event-listener"
+version = "2.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
+
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fastrand"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
+
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "futures"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-intrusive"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5"
+dependencies = [
+ "futures-core",
+ "lock_api",
+ "parking_lot 0.11.2",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
+
+[[package]]
+name = "futures-task"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+
+[[package]]
+name = "futures-util"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "gimli"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+
+[[package]]
+name = "h2"
+version = "0.3.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http",
+ "indexmap 1.9.3",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hashbrown"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+
+[[package]]
+name = "hashlink"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
+dependencies = [
+ "hashbrown 0.11.2",
+]
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "hkdf"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437"
+dependencies = [
+ "hmac",
+]
+
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "http"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "http-range-header"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
+
+[[package]]
+name = "httparse"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
+
+[[package]]
+name = "httpdate"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+
+[[package]]
+name = "hyper"
+version = "0.14.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2 0.4.9",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
+[[package]]
+name = "idna"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "ipnet"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
+
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+
+[[package]]
+name = "js-sys"
+version = "0.3.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.147"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+
+[[package]]
+name = "libz-sys"
+version = "1.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
+
+[[package]]
+name = "lock_api"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+
+[[package]]
+name = "matchit"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+
+[[package]]
+name = "md-5"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "mime"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+dependencies = [
+ "adler",
+]
+
+[[package]]
+name = "mio"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
+dependencies = [
+ "libc",
+ "wasi",
+ "windows-sys",
+]
+
+[[package]]
+name = "multimap"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+
+[[package]]
+name = "native-tls"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi 0.3.2",
+ "libc",
+]
+
+[[package]]
+name = "num_enum"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"
+dependencies = [
+ "num_enum_derive",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "object"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+
+[[package]]
+name = "openssl"
+version = "0.10.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.91"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core 0.8.6",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+dependencies = [
+ "lock_api",
+ "parking_lot_core 0.9.8",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
+dependencies = [
+ "cfg-if",
+ "instant",
+ "libc",
+ "redox_syscall 0.2.16",
+ "smallvec",
+ "winapi",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall 0.3.5",
+ "smallvec",
+ "windows-targets",
+]
+
+[[package]]
+name = "paste"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
+
+[[package]]
+name = "petgraph"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
+dependencies = [
+ "fixedbitset",
+ "indexmap 1.9.3",
+]
+
+[[package]]
+name = "phf"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
+dependencies = [
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "pin-project"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
+
+[[package]]
+name = "postgres-protocol"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d"
+dependencies = [
+ "base64 0.21.2",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "hmac",
+ "md-5",
+ "memchr",
+ "rand",
+ "sha2",
+ "stringprep",
+]
+
+[[package]]
+name = "postgres-types"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6"
+dependencies = [
+ "bytes",
+ "fallible-iterator",
+ "postgres-protocol",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
+[[package]]
+name = "prettyplease"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86"
+dependencies = [
+ "proc-macro2",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "proc-macro-crate"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919"
+dependencies = [
+ "once_cell",
+ "toml_edit",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "prost"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ae5a4388762d5815a9fc0dea33c56b021cdc8dde0c55e0c9ca57197254b0cab"
+dependencies = [
+ "bytes",
+ "cfg-if",
+ "cmake",
+ "heck",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost",
+ "prost-types",
+ "regex",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
+dependencies = [
+ "bytes",
+ "prost",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "rdkafka"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8acd8f5c5482fdf89e8878227bafa442d8c4409f6287391c85549ca83626c27"
+dependencies = [
+ "futures",
+ "libc",
+ "log",
+ "rdkafka-sys",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "slab",
+ "tokio",
+]
+
+[[package]]
+name = "rdkafka-sys"
+version = "3.0.0+1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca35e95c88e08cdc643b25744e38ccee7c93c7e90d1ac6850fe74cbaa40803c3"
+dependencies = [
+ "libc",
+ "libz-sys",
+ "num_enum",
+ "pkg-config",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "redox_users"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+dependencies = [
+ "getrandom",
+ "redox_syscall 0.2.16",
+ "thiserror",
+]
+
+[[package]]
+name = "regex"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+
+[[package]]
+name = "reqwest"
+version = "0.11.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
+dependencies = [
+ "base64 0.21.2",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
+[[package]]
+name = "rustix"
+version = "0.38.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399"
+dependencies = [
+ "bitflags 2.3.3",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
+
+[[package]]
+name = "schannel"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "security-framework"
+version = "2.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.183"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.183"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "server"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "prost",
+ "rdkafka",
+ "reqwest",
+ "serde_json",
+ "sqlx",
+ "tokio",
+ "tokio-postgres",
+ "tonic",
+ "tonic-build",
+]
+
+[[package]]
+name = "sha-1"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "siphasher"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+
+[[package]]
+name = "slab"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
+
+[[package]]
+name = "socket2"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "socket2"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "sqlformat"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4b7922be017ee70900be125523f38bdd644f4f06a1b16e8fa5a8ee8c34bffd4"
+dependencies = [
+ "itertools",
+ "nom",
+ "unicode_categories",
+]
+
+[[package]]
+name = "sqlx"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "551873805652ba0d912fec5bbb0f8b4cdd96baf8e2ebf5970e5671092966019b"
+dependencies = [
+ "sqlx-core",
+ "sqlx-macros",
+]
+
+[[package]]
+name = "sqlx-core"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5"
+dependencies = [
+ "ahash",
+ "atoi",
+ "base64 0.13.1",
+ "bitflags 1.3.2",
+ "byteorder",
+ "bytes",
+ "crc",
+ "crossbeam-queue",
+ "dirs",
+ "either",
+ "event-listener",
+ "futures-channel",
+ "futures-core",
+ "futures-intrusive",
+ "futures-util",
+ "hashlink",
+ "hex",
+ "hkdf",
+ "hmac",
+ "indexmap 1.9.3",
+ "itoa",
+ "libc",
+ "log",
+ "md-5",
+ "memchr",
+ "once_cell",
+ "paste",
+ "percent-encoding",
+ "rand",
+ "serde",
+ "serde_json",
+ "sha-1",
+ "sha2",
+ "smallvec",
+ "sqlformat",
+ "sqlx-rt",
+ "stringprep",
+ "thiserror",
+ "tokio-stream",
+ "url",
+ "whoami",
+]
+
+[[package]]
+name = "sqlx-macros"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc0fba2b0cae21fc00fe6046f8baa4c7fcb49e379f0f592b04696607f69ed2e1"
+dependencies = [
+ "dotenv",
+ "either",
+ "heck",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "sha2",
+ "sqlx-core",
+ "sqlx-rt",
+ "syn 1.0.109",
+ "url",
+]
+
+[[package]]
+name = "sqlx-rt"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4db708cd3e459078f85f39f96a00960bd841f66ee2a669e90bf36907f5a79aae"
+dependencies = [
+ "native-tls",
+ "once_cell",
+ "tokio",
+ "tokio-native-tls",
+]
+
+[[package]]
+name = "stringprep"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "subtle"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
+[[package]]
+name = "tempfile"
+version = "3.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "redox_syscall 0.3.5",
+ "rustix",
+ "windows-sys",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "tokio"
+version = "1.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
+dependencies = [
+ "autocfg",
+ "backtrace",
+ "bytes",
+ "libc",
+ "mio",
+ "num_cpus",
+ "parking_lot 0.12.1",
+ "pin-project-lite",
+ "signal-hook-registry",
+ "socket2 0.4.9",
+ "tokio-macros",
+ "windows-sys",
+]
+
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-postgres"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "bytes",
+ "fallible-iterator",
+ "futures-channel",
+ "futures-util",
+ "log",
+ "parking_lot 0.12.1",
+ "percent-encoding",
+ "phf",
+ "pin-project-lite",
+ "postgres-protocol",
+ "postgres-types",
+ "socket2 0.5.3",
+ "tokio",
+ "tokio-util",
+]
+
+[[package]]
+name = "tokio-stream"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
+
+[[package]]
+name = "toml_edit"
+version = "0.19.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
+dependencies = [
+ "indexmap 2.0.0",
+ "toml_datetime",
+ "winnow",
+]
+
+[[package]]
+name = "tonic"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum",
+ "base64 0.13.1",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "prost-derive",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1"
+dependencies = [
+ "prettyplease",
+ "proc-macro2",
+ "prost-build",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap 1.9.3",
+ "pin-project",
+ "pin-project-lite",
+ "rand",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
+dependencies = [
+ "bitflags 1.3.2",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-range-header",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
+[[package]]
+name = "tracing"
+version = "0.1.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+dependencies = [
+ "cfg-if",
+ "log",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "tracing-futures"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
+dependencies = [
+ "pin-project",
+ "tracing",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+
+[[package]]
+name = "typenum"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
+[[package]]
+name = "url"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+]
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
+
+[[package]]
+name = "web-sys"
+version = "0.3.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "which"
+version = "4.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
+dependencies = [
+ "either",
+ "libc",
+ "once_cell",
+]
+
+[[package]]
+name = "whoami"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50"
+dependencies = [
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
+[[package]]
+name = "winnow"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "winreg"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
+dependencies = [
+ "winapi",
+]
diff --git a/integration_tests/feature-store/server/Cargo.toml b/integration_tests/feature-store/server/Cargo.toml
new file mode 100644
index 0000000000000..c1fe08fb15475
--- /dev/null
+++ b/integration_tests/feature-store/server/Cargo.toml
@@ -0,0 +1,28 @@
+[workspace]
+members = []
+
+[package]
+name = "server"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+sqlx = { version = "0.5", features = [ "runtime-tokio-native-tls" , "postgres" ] }
+tokio = { version = "1", features = ["full"] }
+tonic = "0.7.1"
+reqwest = { version = "0.11", features = ["blocking"] }
+rdkafka = { version = "0.25", features = ["dynamic-linking"] }
+serde_json = "1.0"
+prost = "0.10"
+clap = "2.26.0"
+tokio-postgres = "0.7.8"
+tonic-build = "0.7.1"
+
+[build-dependencies]
+tonic-build = "0.7.1"
+
+[[bin]]
+name = "server"
+path = "src/main.rs"
diff --git a/integration_tests/feature-store/server/actor.proto b/integration_tests/feature-store/server/actor.proto
new file mode 100644
index 0000000000000..2253fce4f20ed
--- /dev/null
+++ b/integration_tests/feature-store/server/actor.proto
@@ -0,0 +1,67 @@
+syntax = "proto3";
+package server_pb;
+
+service Server {
+    rpc GetFeature(GetFeatureRequest) returns (GetFeatureResponse) {}
+    rpc ReportAction(ReportActionRequest) returns (ReportActionResponse) {}
+    rpc ReportTaxiAction(ReportTaxiActionRequest) returns (ReportTaxiActionResponse) {}
+    rpc GetTaxiAmount(GetTaxiAmountRequest) returns (GetTaxiAmountResponse) {}
+    rpc StartTraining(StartTrainingRequest) returns (StartTrainingResponse) {}
+}
+
+message ReportActionRequest {
+    string userid = 1;
+    string eventtype = 2;
+    int64 changenum = 3;
+}
+
+message ReportActionResponse {
+    uint64 timestamp = 1;
+}
+
+message GetFeatureRequest {
+    string userid = 1;
+}
+
+message GetFeatureResponse {
+    uint64 count = 1;
+    int64 sum = 2;
+}
+
+message ReportTaxiActionRequest {
+    int32 VendorID = 1;
+    string lpep_pickup_datetime = 2;
+    string lpep_dropoff_datetime = 3;
+    bool store_and_fwd_flag = 4;
+    double RatecodeID = 5;
+    int64 PULocationID = 6;
+    int64 DOLocationID = 7;
+    double passenger_count = 8;
+    double trip_distance = 9;
+    double fare_amount = 10;
+    double extra = 11;
+    double mta_tax = 12;
+    double tip_amount = 13;
+    double tolls_amount = 14;
+    double ehail_fee = 15;
+    double improvement_surcharge = 16;
+    double total_amount = 17;
+    double payment_type = 18;
+    double trip_type = 19;
+    double congestion_surcharge = 20;
+}
+
+message ReportTaxiActionResponse {}
+
+message GetTaxiAmountRequest {
+    int64 DOLocationID = 1;
+    int64 PULocationID = 2;
+}
+
+message GetTaxiAmountResponse {
+    double fare_amount = 1;
+}
+
+message StartTrainingRequest{}
+
+message StartTrainingResponse{}
\ No newline at end of file
diff --git a/integration_tests/feature-store/server/build.rs b/integration_tests/feature-store/server/build.rs
new file mode 100644
index 0000000000000..1dae5151acdb1
--- /dev/null
+++ b/integration_tests/feature-store/server/build.rs
@@ -0,0 +1,20 @@
+use std::fs;
+
+fn main() {
+    let actor_proto = "./actor.proto";
+    let model_proto = "./model/model.proto";
+
+    tonic_build::configure()
+        .build_server(true)
+        .out_dir("./src")
+        .compile(&[actor_proto], &["."])
+        .unwrap_or_else(|e| panic!("protobuf compile error: {}", e));
+    fs::copy("./src/server_pb.rs", "../simulator/src/server_pb.rs").unwrap();
+
+    tonic_build::configure()
+        .build_client(true)
+        .out_dir("./src")
+        .compile(&[model_proto], &["."])
+        .unwrap_or_else(|e| panic!("protobuf compile error: {}", e));
+    println!("cargo:rerun-if-changed={}", actor_proto);
+}
diff --git a/integration_tests/feature-store/server/model/__init__.py b/integration_tests/feature-store/server/model/__init__.py
new file mode 100644
index 0000000000000..29863aba2770c
--- /dev/null
+++ b/integration_tests/feature-store/server/model/__init__.py
@@ -0,0 +1,9 @@
+import platform
+import os
+from distutils import util
+
+# check environment
+if 'arm64' in platform.machine() and 'mac' in util.get_platform():
+    print("Found m1 chip")
+    os.environ["GRPC_PYTHON_BUILD_SYSTEM_OPENSSL"] = "1"
+    os.environ["GRPC_PYTHON_BUILD_SYSTEM_ZLIB"] = "1"
diff --git a/integration_tests/feature-store/server/model/__main__.py b/integration_tests/feature-store/server/model/__main__.py
new file mode 100644
index 0000000000000..09b82dfc740e1
--- /dev/null
+++ b/integration_tests/feature-store/server/model/__main__.py
@@ -0,0 +1,6 @@
+import __init__
+import model
+
+if __name__ == "__main__":
+    with model.TrainingModelService() as servicer:
+        servicer.serve()
diff --git a/integration_tests/feature-store/server/model/model.proto b/integration_tests/feature-store/server/model/model.proto
new file mode 100644
index 0000000000000..0ffc1de602710
--- /dev/null
+++ b/integration_tests/feature-store/server/model/model.proto
@@ -0,0 +1,22 @@
+syntax = "proto3";
+package model;
+
+service Model {
+  rpc GetAmount(GetAmountRequest) returns (GetAmountResponse) {}
+  rpc Training(TrainingRequest) returns (TrainingResponse) {}
+}
+
+message TrainingRequest {
+}
+
+message TrainingResponse {
+}
+
+message GetAmountRequest {
+  int64 do_location_id = 1;
+  int64 pu_location_id = 2;
+}
+
+message GetAmountResponse {
+  float amount = 1;
+}
diff --git a/integration_tests/feature-store/server/model/model.py b/integration_tests/feature-store/server/model/model.py
new file mode 100644
index 0000000000000..771fab6eaa7bc
--- /dev/null
+++ b/integration_tests/feature-store/server/model/model.py
@@ -0,0 +1,96 @@
+import json
+import os
+from concurrent import futures
+import ast
+import numpy as np
+import traceback
+from model_pb2_grpc import *
+import psycopg
+import sql
+import model_pb2_grpc
+import pandas as pd
+from psycopg import sql as pgsql
+from sklearn.ensemble import GradientBoostingRegressor
+from model_pb2 import TrainingResponse,GetAmountResponse
+
+"""
+accompanied modules are generated by calling the following command
+
+```
+python3 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. model.proto
+```
+
+"""
+class TrainingModelService(ModelServicer):
+    def __init__(self):
+        super(TrainingModelService, self).__init__()
+        self.model = GradientBoostingRegressor()
+        self.conn = psycopg.connect("dbname=dev user=root host=frontend-node-0 port=4566")
+    def Training(self, request, context):
+        print(f"get data for training!")
+        try:
+            with self.conn.cursor() as cur:
+                cur.execute(sql.GET_BATCH_FOR_TRAINING)
+                results = cur.fetchall()
+                df = pd.DataFrame(list(results))
+                train_y = df.loc[:, 0]
+                train_x = df.drop(columns=[0,1,2,14,15])
+                print(f"training!")
+                self.model.fit(train_x, train_y)
+                return TrainingResponse()
+        except Exception as e:
+            print(traceback.format_exc())
+            context.set_code(grpc.StatusCode.INTERNAL)
+            context.set_details(e)
+            return TrainingResponse()
+
+    def GetAmount(self, request, context):
+        do_location_id = request.do_location_id
+        pu_location_id = request.pu_location_id
+        try:
+            with self.conn.cursor() as cur:
+                cur.execute(sql.GET_FEATURE_DO_LOCATION % do_location_id)
+                results = cur.fetchall()
+                df = pd.DataFrame(list(results))
+                train_x1 = df.drop(columns=[0,1])
+                cur.execute(sql.GET_FEATURE_PU_LOCATION % pu_location_id)
+                results = cur.fetchall()
+                df = pd.DataFrame(list(results))
+                train_x2 = df.drop(columns=[0,1])
+                train_x = pd.concat([train_x1, train_x2], axis=1)
+                result = self.model.predict(train_x)
+                return GetAmountResponse(amount = result)
+        except Exception as e:
+            print(traceback.format_exc())
+            context.set_code(grpc.StatusCode.INTERNAL)
+            context.set_details(e)
+            return GetAmountResponse(amount = 0.0)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.conn.close()
+
+    def serve(self):
+        server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        model_pb2_grpc.add_ModelServicer_to_server(
+            self, server)
+        server.add_insecure_port('[::]:8080')
+        server.start()
+        server.wait_for_termination()
+
+if __name__ == '__main__':
+    for i in range(1):
+        print("making the", i, "th connection")
+
+        conn = psycopg.connect("dbname=dev user=root host=frontend-node-0 port=4566")
+        with conn.cursor() as cur:
+            # the following code will panic
+            # because placeholder is completely not supported
+            # even varchar parsed to TypeOid 0, which isn't even defined
+            cur.execute("select (2, %s);", ("2333333",))
+            results = cur.fetchall()
+        print(results)
+
+        conn.close()
\ No newline at end of file
diff --git a/integration_tests/feature-store/server/model/model_pb2.py b/integration_tests/feature-store/server/model/model_pb2.py
new file mode 100644
index 0000000000000..83a1d2dc779bb
--- /dev/null
+++ b/integration_tests/feature-store/server/model/model_pb2.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: model.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bmodel.proto\x12\x05model\"\x11\n\x0fTrainingRequest\"\x12\n\x10TrainingResponse\"B\n\x10GetAmountRequest\x12\x16\n\x0e\x64o_location_id\x18\x01 \x01(\x03\x12\x16\n\x0epu_location_id\x18\x02 \x01(\x03\"#\n\x11GetAmountResponse\x12\x0e\n\x06\x61mount\x18\x01 \x01(\x02\x32\x88\x01\n\x05Model\x12@\n\tGetAmount\x12\x17.model.GetAmountRequest\x1a\x18.model.GetAmountResponse\"\x00\x12=\n\x08Training\x12\x16.model.TrainingRequest\x1a\x17.model.TrainingResponse\"\x00\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'model_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  _globals['_TRAININGREQUEST']._serialized_start=22
+  _globals['_TRAININGREQUEST']._serialized_end=39
+  _globals['_TRAININGRESPONSE']._serialized_start=41
+  _globals['_TRAININGRESPONSE']._serialized_end=59
+  _globals['_GETAMOUNTREQUEST']._serialized_start=61
+  _globals['_GETAMOUNTREQUEST']._serialized_end=127
+  _globals['_GETAMOUNTRESPONSE']._serialized_start=129
+  _globals['_GETAMOUNTRESPONSE']._serialized_end=164
+  _globals['_MODEL']._serialized_start=167
+  _globals['_MODEL']._serialized_end=303
+# @@protoc_insertion_point(module_scope)
diff --git a/integration_tests/feature-store/server/model/model_pb2_grpc.py b/integration_tests/feature-store/server/model/model_pb2_grpc.py
new file mode 100644
index 0000000000000..4c9d12be48b6d
--- /dev/null
+++ b/integration_tests/feature-store/server/model/model_pb2_grpc.py
@@ -0,0 +1,99 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import model_pb2 as model__pb2
+
+
+class ModelStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.GetAmount = channel.unary_unary(
+                '/model.Model/GetAmount',
+                request_serializer=model__pb2.GetAmountRequest.SerializeToString,
+                response_deserializer=model__pb2.GetAmountResponse.FromString,
+                )
+        self.Training = channel.unary_unary(
+                '/model.Model/Training',
+                request_serializer=model__pb2.TrainingRequest.SerializeToString,
+                response_deserializer=model__pb2.TrainingResponse.FromString,
+                )
+
+
+class ModelServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def GetAmount(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Training(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ModelServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'GetAmount': grpc.unary_unary_rpc_method_handler(
+                    servicer.GetAmount,
+                    request_deserializer=model__pb2.GetAmountRequest.FromString,
+                    response_serializer=model__pb2.GetAmountResponse.SerializeToString,
+            ),
+            'Training': grpc.unary_unary_rpc_method_handler(
+                    servicer.Training,
+                    request_deserializer=model__pb2.TrainingRequest.FromString,
+                    response_serializer=model__pb2.TrainingResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'model.Model', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Model(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def GetAmount(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/model.Model/GetAmount',
+            model__pb2.GetAmountRequest.SerializeToString,
+            model__pb2.GetAmountResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Training(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/model.Model/Training',
+            model__pb2.TrainingRequest.SerializeToString,
+            model__pb2.TrainingResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/integration_tests/feature-store/server/model/requirements.txt b/integration_tests/feature-store/server/model/requirements.txt
new file mode 100644
index 0000000000000..bc40c0ead3fbe
--- /dev/null
+++ b/integration_tests/feature-store/server/model/requirements.txt
@@ -0,0 +1,6 @@
+grpcio==1.48.0
+numpy==1.21.4
+protobuf==4.21.5
+psycopg==3.0.16
+scikit-learn==1.3.0
+pandas==1.4.0
diff --git a/integration_tests/feature-store/server/model/sql.py b/integration_tests/feature-store/server/model/sql.py
new file mode 100644
index 0000000000000..6eaafa87394b8
--- /dev/null
+++ b/integration_tests/feature-store/server/model/sql.py
@@ -0,0 +1,14 @@
+GET_BATCH_FOR_TRAINING = """
+select  l.fare_amount,d.*, p.*
+from location_id_store l
+join converted_features_with_do_location d on l.window_start = d.window_start and l.do_location_id = d.do_location_id
+join converted_features_with_pu_location p on l.window_start = p.window_start and l.pu_location_id = p.pu_location_id;
+"""
+
+GET_FEATURE_DO_LOCATION = """
+select * from converted_features_with_do_location where do_location_id=%s order by window_start desc limit 1;
+"""
+
+GET_FEATURE_PU_LOCATION = """
+select * from converted_features_with_pu_location where pu_location_id=%s order by window_start desc limit 1;
+"""
\ No newline at end of file
diff --git a/integration_tests/feature-store/server/src/feature_store.rs b/integration_tests/feature-store/server/src/feature_store.rs
new file mode 100644
index 0000000000000..883a7c332767d
--- /dev/null
+++ b/integration_tests/feature-store/server/src/feature_store.rs
@@ -0,0 +1,119 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use tonic::{Request, Response, Status};
+
+use crate::kafka::KafkaSink;
+use crate::server_pb::server_server::Server;
+use crate::server_pb::{
+    GetFeatureRequest, GetFeatureResponse, GetTaxiAmountRequest, GetTaxiAmountResponse,
+    ReportActionRequest, ReportActionResponse, ReportTaxiActionRequest, ReportTaxiActionResponse,
+    StartTrainingRequest, StartTrainingResponse,
+};
+
+pub struct FeatureStoreServer {
+    pub(crate) kafka: KafkaSink,
+}
+
+#[tonic::async_trait]
+impl Server for FeatureStoreServer {
+    async fn get_feature(
+        &self,
+        request: Request<GetFeatureRequest>,
+    ) -> Result<Response<GetFeatureResponse>, Status> {
+        let userid = request.into_inner().userid;
+        println!("MFA: get_feature: userid={}", userid);
+        let (count, sum) = self.get_mfa_feature_from_rw(userid.clone()).await.unwrap();
+        Ok(Response::new(GetFeatureResponse {
+            count: count,
+            sum: sum,
+        }))
+    }
+
+    async fn report_action(
+        &self,
+        request: Request<ReportActionRequest>,
+    ) -> Result<Response<ReportActionResponse>, Status> {
+        let message = request.into_inner();
+        self.mock_report_action(&message).await
+    }
+
+    async fn report_taxi_action(
+        &self,
+        request: tonic::Request<ReportTaxiActionRequest>,
+    ) -> Result<tonic::Response<ReportTaxiActionResponse>, tonic::Status> {
+        let message = request.into_inner();
+        self.mock_report_taxi_action(&message).await
+    }
+
+    async fn start_training(
+        &self,
+        _request: tonic::Request<StartTrainingRequest>,
+    ) -> Result<tonic::Response<StartTrainingResponse>, tonic::Status> {
+        self.do_training().await
+    }
+
+    async fn get_taxi_amount(
+        &self,
+        request: tonic::Request<GetTaxiAmountRequest>,
+    ) -> Result<tonic::Response<GetTaxiAmountResponse>, tonic::Status> {
+        let do_location_id = request.into_inner();
+        let fare_amount = self.get_taxi_amount(do_location_id.do_location_id.clone(),do_location_id.pu_location_id.clone()).await.unwrap();
+        Ok(Response::new(GetTaxiAmountResponse {
+            fare_amount: fare_amount as f64,
+        }))
+    }
+}
+
+impl FeatureStoreServer {
+    async fn mock_report_action(
+        &self,
+        message: &ReportActionRequest,
+    ) -> Result<Response<ReportActionResponse>, Status> {
+        let duration = SystemTime::now().duration_since(UNIX_EPOCH);
+        match duration {
+            Ok(dur) => {
+                let timestamp = dur.as_micros();
+                let json = Self::create_sink_mfa_json(message, timestamp as u64);
+                println!("timestamp: {}, payload: {}", timestamp, json.clone());
+                self.kafka.send("0".to_string(), json).await;
+                Ok(Response::new(ReportActionResponse {
+                    timestamp: timestamp as u64,
+                }))
+            }
+            Err(_e) => Err(Status::unknown("Failed to generate timestamp".to_string())),
+        }
+    }
+
+    pub(crate) fn create_sink_mfa_json(message: &ReportActionRequest, timestamp: u64) -> String {
+        format!(
+            "{{\"userid\": {:?}, \"eventype\": {:?}, \"changenum\": {:?}, \"timestamp\": {:?}}}",
+            message.userid, message.eventtype, message.changenum, timestamp
+        )
+        .to_string()
+    }
+
+    async fn mock_report_taxi_action(
+        &self,
+        message: &ReportTaxiActionRequest,
+    ) -> Result<Response<ReportTaxiActionResponse>, Status> {
+        let json = Self::create_sink_taxi_json(message);
+        self.kafka.send("0".to_string(), json).await;
+        Ok(Response::new(ReportTaxiActionResponse {}))
+    }
+
+    pub(crate) fn create_sink_taxi_json(message: &ReportTaxiActionRequest) -> String {
+        format!(
+            "{{\"vendor_id\": {:?}, \"lpep_pickup_datetime\": {:?}, \"lpep_dropoff_datetime\": {:?}, \"store_and_fwd_flag\": {:?},
+            \"ratecode_id\": {:?}, \"pu_location_id\": {:?}, \"do_location_id\": {:?}, \"passenger_count\": {:?},
+            \"trip_distance\": {:?}, \"fare_amount\": {:?}, \"extra\": {:?}, \"mta_tax\": {:?},
+            \"tip_amount\": {:?}, \"tolls_amount\": {:?}, \"ehail_fee\": {:?}, \"improvement_surcharge\": {:?},
+            \"total_amount\": {:?}, \"payment_type\": {:?}, \"trip_type\": {:?}, \"congestion_surcharge\": {:?}}}",
+            message.vendor_id,message.lpep_pickup_datetime,message.lpep_dropoff_datetime,message.store_and_fwd_flag,
+            message.ratecode_id,message.pu_location_id,message.do_location_id,message.passenger_count,
+            message.trip_distance,message.fare_amount,message.extra,message.mta_tax,
+            message.tip_amount,message.tolls_amount,message.ehail_fee,message.improvement_surcharge,
+            message.total_amount,message.payment_type,message.trip_type,message.congestion_surcharge
+        )
+        .to_string()
+    }
+}
diff --git a/integration_tests/feature-store/server/src/kafka.rs b/integration_tests/feature-store/server/src/kafka.rs
new file mode 100644
index 0000000000000..05b3ecb0ca3ed
--- /dev/null
+++ b/integration_tests/feature-store/server/src/kafka.rs
@@ -0,0 +1,75 @@
+use std::time::Duration;
+
+use rdkafka::consumer::{Consumer, ConsumerContext, StreamConsumer};
+use rdkafka::producer::{FutureProducer, FutureRecord};
+use rdkafka::{ClientConfig, ClientContext, Message};
+
+struct CustomContext;
+
+impl ClientContext for CustomContext {}
+
+impl ConsumerContext for CustomContext {}
+
+// A type alias with your custom consumer can be created for convenience.
+type LoggingConsumer = StreamConsumer<CustomContext>;
+
+pub struct KafkaSink {
+    client_config: FutureProducer,
+    output_topic: String,
+}
+
+impl KafkaSink {
+    pub(crate) fn new(brokers: String, output_topic: String) -> KafkaSink {
+        KafkaSink {
+            client_config: ClientConfig::new()
+                .set("group.id", "feature-store")
+                .set("bootstrap.servers", brokers.clone())
+                .set("queue.buffering.max.ms", "0") // Do not buffer
+                .create()
+                .expect("Producer creation failed"),
+            output_topic,
+        }
+    }
+
+    pub async fn send(&self, record_id: String, payload: String) {
+        let output_topic = self.output_topic.clone();
+        let record = FutureRecord::to(&*output_topic)
+            .payload(payload.as_bytes())
+            .key(record_id.as_bytes());
+        self.client_config
+            .send(record, Duration::from_secs(1))
+            .await
+            .expect("Failed to create send message request");
+    }
+
+    pub async fn mock_consume() {
+        let consumer: LoggingConsumer = ClientConfig::new()
+            .set("group.id", "feature-store")
+            .set("bootstrap.servers", "kafka:9092")
+            .create_with_context(CustomContext)
+            .expect("Failed to create consumer");
+        consumer
+            .subscribe(&vec!["taxi"])
+            .expect("Failed to subscribe");
+        println!("Ready to consume");
+
+        loop {
+            match consumer
+                .recv()
+                .await
+                .expect("Failed to poll")
+                .payload_view::<str>()
+            {
+                Some(Ok(payload)) => {
+                    println!("Received message: {}", payload);
+                }
+                Some(Err(e)) => {
+                    println!("Failed to decode message: {}", e);
+                }
+                None => {
+                    println!("Received empty message");
+                }
+            }
+        }
+    }
+}
diff --git a/integration_tests/feature-store/server/src/main.rs b/integration_tests/feature-store/server/src/main.rs
new file mode 100644
index 0000000000000..adab174aaa227
--- /dev/null
+++ b/integration_tests/feature-store/server/src/main.rs
@@ -0,0 +1,63 @@
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+use clap::{App, Arg, ArgMatches};
+
+use crate::feature_store::FeatureStoreServer;
+use crate::kafka::KafkaSink;
+use crate::server_pb::server_server::ServerServer;
+
+mod feature_store;
+mod kafka;
+mod model;
+mod server_pb;
+mod serving;
+
+#[tokio::main]
+async fn main() {
+    println!("Reading args");
+    let args = get_args();
+    let kafka_sink = KafkaSink::new(
+        args.value_of("brokers")
+            .expect("failed to decode brokers")
+            .to_string(),
+        args.value_of("output-topic")
+            .expect("failed to decode output_topics")
+            .to_string(),
+    );
+    println!("Testing Kafka payload,args{:?}",args);
+    tokio::spawn(KafkaSink::mock_consume());
+    kafka_sink
+        .send("0".to_string(), "{init: true}".to_string())
+        .await;
+    let server = ServerServer::new(FeatureStoreServer { kafka: kafka_sink });
+
+    tonic::transport::Server::builder()
+        .add_service(server)
+        .serve(SocketAddr::new(
+            IpAddr::from(Ipv4Addr::new(127, 0, 0, 1)),
+            2666,
+        ))
+        .await
+        .unwrap()
+}
+
+fn get_args<'a>() -> ArgMatches<'a> {
+    App::new("feature-store")
+        .about("Feature store")
+        .arg(
+            Arg::with_name("brokers")
+                .short("b")
+                .long("brokers")
+                .help("Kafka broker list")
+                .takes_value(true)
+                .default_value("kafka:9092"),
+        )
+        .arg(
+            Arg::with_name("output-topic")
+                .long("output-topics")
+                .help("Output topics names")
+                .default_value("taxi")
+                .takes_value(true),
+        )
+        .get_matches()
+}
diff --git a/integration_tests/feature-store/server/src/model.rs b/integration_tests/feature-store/server/src/model.rs
new file mode 100644
index 0000000000000..2b141fafce0d1
--- /dev/null
+++ b/integration_tests/feature-store/server/src/model.rs
@@ -0,0 +1,292 @@
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TrainingRequest {
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TrainingResponse {
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetAmountRequest {
+    #[prost(int64, tag="1")]
+    pub do_location_id: i64,
+    #[prost(int64, tag="2")]
+    pub pu_location_id: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetAmountResponse {
+    #[prost(float, tag="1")]
+    pub amount: f32,
+}
+/// Generated client implementations.
+pub mod model_client {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    #[derive(Debug, Clone)]
+    pub struct ModelClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl ModelClient<tonic::transport::Channel> {
+        /// Attempt to create a new client by connecting to a given endpoint.
+        pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: std::convert::TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+            Ok(Self::new(conn))
+        }
+    }
+    impl<T> ModelClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::BoxBody>,
+        T::Error: Into<StdError>,
+        T::ResponseBody: Body<Data = Bytes> + Send + 'static,
+        <T::ResponseBody as Body>::Error: Into<StdError> + Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> ModelClient<InterceptedService<T, F>>
+        where
+            F: tonic::service::Interceptor,
+            T::ResponseBody: Default,
+            T: tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+                Response = http::Response<
+                    <T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
+                >,
+            >,
+            <T as tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+            >>::Error: Into<StdError> + Send + Sync,
+        {
+            ModelClient::new(InterceptedService::new(inner, interceptor))
+        }
+        /// Compress requests with `gzip`.
+        ///
+        /// This requires the server to support it otherwise it might respond with an
+        /// error.
+        #[must_use]
+        pub fn send_gzip(mut self) -> Self {
+            self.inner = self.inner.send_gzip();
+            self
+        }
+        /// Enable decompressing responses with `gzip`.
+        #[must_use]
+        pub fn accept_gzip(mut self) -> Self {
+            self.inner = self.inner.accept_gzip();
+            self
+        }
+        pub async fn get_amount(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetAmountRequest>,
+        ) -> Result<tonic::Response<super::GetAmountResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/model.Model/GetAmount");
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn training(
+            &mut self,
+            request: impl tonic::IntoRequest<super::TrainingRequest>,
+        ) -> Result<tonic::Response<super::TrainingResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/model.Model/Training");
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+    }
+}
+/// Generated server implementations.
+pub mod model_server {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    ///Generated trait containing gRPC methods that should be implemented for use with ModelServer.
+    #[async_trait]
+    pub trait Model: Send + Sync + 'static {
+        async fn get_amount(
+            &self,
+            request: tonic::Request<super::GetAmountRequest>,
+        ) -> Result<tonic::Response<super::GetAmountResponse>, tonic::Status>;
+        async fn training(
+            &self,
+            request: tonic::Request<super::TrainingRequest>,
+        ) -> Result<tonic::Response<super::TrainingResponse>, tonic::Status>;
+    }
+    #[derive(Debug)]
+    pub struct ModelServer<T: Model> {
+        inner: _Inner<T>,
+        accept_compression_encodings: (),
+        send_compression_encodings: (),
+    }
+    struct _Inner<T>(Arc<T>);
+    impl<T: Model> ModelServer<T> {
+        pub fn new(inner: T) -> Self {
+            Self::from_arc(Arc::new(inner))
+        }
+        pub fn from_arc(inner: Arc<T>) -> Self {
+            let inner = _Inner(inner);
+            Self {
+                inner,
+                accept_compression_encodings: Default::default(),
+                send_compression_encodings: Default::default(),
+            }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> InterceptedService<Self, F>
+        where
+            F: tonic::service::Interceptor,
+        {
+            InterceptedService::new(Self::new(inner), interceptor)
+        }
+    }
+    impl<T, B> tonic::codegen::Service<http::Request<B>> for ModelServer<T>
+    where
+        T: Model,
+        B: Body + Send + 'static,
+        B::Error: Into<StdError> + Send + 'static,
+    {
+        type Response = http::Response<tonic::body::BoxBody>;
+        type Error = std::convert::Infallible;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(
+            &mut self,
+            _cx: &mut Context<'_>,
+        ) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, req: http::Request<B>) -> Self::Future {
+            let inner = self.inner.clone();
+            match req.uri().path() {
+                "/model.Model/GetAmount" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetAmountSvc<T: Model>(pub Arc<T>);
+                    impl<T: Model> tonic::server::UnaryService<super::GetAmountRequest>
+                    for GetAmountSvc<T> {
+                        type Response = super::GetAmountResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::GetAmountRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move { (*inner).get_amount(request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetAmountSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/model.Model/Training" => {
+                    #[allow(non_camel_case_types)]
+                    struct TrainingSvc<T: Model>(pub Arc<T>);
+                    impl<T: Model> tonic::server::UnaryService<super::TrainingRequest>
+                    for TrainingSvc<T> {
+                        type Response = super::TrainingResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::TrainingRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move { (*inner).training(request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = TrainingSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                _ => {
+                    Box::pin(async move {
+                        Ok(
+                            http::Response::builder()
+                                .status(200)
+                                .header("grpc-status", "12")
+                                .header("content-type", "application/grpc")
+                                .body(empty_body())
+                                .unwrap(),
+                        )
+                    })
+                }
+            }
+        }
+    }
+    impl<T: Model> Clone for ModelServer<T> {
+        fn clone(&self) -> Self {
+            let inner = self.inner.clone();
+            Self {
+                inner,
+                accept_compression_encodings: self.accept_compression_encodings,
+                send_compression_encodings: self.send_compression_encodings,
+            }
+        }
+    }
+    impl<T: Model> Clone for _Inner<T> {
+        fn clone(&self) -> Self {
+            Self(self.0.clone())
+        }
+    }
+    impl<T: std::fmt::Debug> std::fmt::Debug for _Inner<T> {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "{:?}", self.0)
+        }
+    }
+    impl<T: Model> tonic::transport::NamedService for ModelServer<T> {
+        const NAME: &'static str = "model.Model";
+    }
+}
diff --git a/integration_tests/feature-store/server/src/server_pb.rs b/integration_tests/feature-store/server/src/server_pb.rs
new file mode 100644
index 0000000000000..1f2912d868412
--- /dev/null
+++ b/integration_tests/feature-store/server/src/server_pb.rs
@@ -0,0 +1,561 @@
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportActionRequest {
+    #[prost(string, tag="1")]
+    pub userid: ::prost::alloc::string::String,
+    #[prost(string, tag="2")]
+    pub eventtype: ::prost::alloc::string::String,
+    #[prost(int64, tag="3")]
+    pub changenum: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportActionResponse {
+    #[prost(uint64, tag="1")]
+    pub timestamp: u64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetFeatureRequest {
+    #[prost(string, tag="1")]
+    pub userid: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetFeatureResponse {
+    #[prost(uint64, tag="1")]
+    pub count: u64,
+    #[prost(int64, tag="2")]
+    pub sum: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportTaxiActionRequest {
+    #[prost(int32, tag="1")]
+    pub vendor_id: i32,
+    #[prost(string, tag="2")]
+    pub lpep_pickup_datetime: ::prost::alloc::string::String,
+    #[prost(string, tag="3")]
+    pub lpep_dropoff_datetime: ::prost::alloc::string::String,
+    #[prost(bool, tag="4")]
+    pub store_and_fwd_flag: bool,
+    #[prost(double, tag="5")]
+    pub ratecode_id: f64,
+    #[prost(int64, tag="6")]
+    pub pu_location_id: i64,
+    #[prost(int64, tag="7")]
+    pub do_location_id: i64,
+    #[prost(double, tag="8")]
+    pub passenger_count: f64,
+    #[prost(double, tag="9")]
+    pub trip_distance: f64,
+    #[prost(double, tag="10")]
+    pub fare_amount: f64,
+    #[prost(double, tag="11")]
+    pub extra: f64,
+    #[prost(double, tag="12")]
+    pub mta_tax: f64,
+    #[prost(double, tag="13")]
+    pub tip_amount: f64,
+    #[prost(double, tag="14")]
+    pub tolls_amount: f64,
+    #[prost(double, tag="15")]
+    pub ehail_fee: f64,
+    #[prost(double, tag="16")]
+    pub improvement_surcharge: f64,
+    #[prost(double, tag="17")]
+    pub total_amount: f64,
+    #[prost(double, tag="18")]
+    pub payment_type: f64,
+    #[prost(double, tag="19")]
+    pub trip_type: f64,
+    #[prost(double, tag="20")]
+    pub congestion_surcharge: f64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportTaxiActionResponse {
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetTaxiAmountRequest {
+    #[prost(int64, tag="1")]
+    pub do_location_id: i64,
+    #[prost(int64, tag="2")]
+    pub pu_location_id: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetTaxiAmountResponse {
+    #[prost(double, tag="1")]
+    pub fare_amount: f64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct StartTrainingRequest {
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct StartTrainingResponse {
+}
+/// Generated client implementations.
+pub mod server_client {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    #[derive(Debug, Clone)]
+    pub struct ServerClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl ServerClient<tonic::transport::Channel> {
+        /// Attempt to create a new client by connecting to a given endpoint.
+        pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: std::convert::TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+            Ok(Self::new(conn))
+        }
+    }
+    impl<T> ServerClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::BoxBody>,
+        T::Error: Into<StdError>,
+        T::ResponseBody: Body<Data = Bytes> + Send + 'static,
+        <T::ResponseBody as Body>::Error: Into<StdError> + Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> ServerClient<InterceptedService<T, F>>
+        where
+            F: tonic::service::Interceptor,
+            T::ResponseBody: Default,
+            T: tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+                Response = http::Response<
+                    <T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
+                >,
+            >,
+            <T as tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+            >>::Error: Into<StdError> + Send + Sync,
+        {
+            ServerClient::new(InterceptedService::new(inner, interceptor))
+        }
+        /// Compress requests with `gzip`.
+        ///
+        /// This requires the server to support it otherwise it might respond with an
+        /// error.
+        #[must_use]
+        pub fn send_gzip(mut self) -> Self {
+            self.inner = self.inner.send_gzip();
+            self
+        }
+        /// Enable decompressing responses with `gzip`.
+        #[must_use]
+        pub fn accept_gzip(mut self) -> Self {
+            self.inner = self.inner.accept_gzip();
+            self
+        }
+        pub async fn get_feature(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetFeatureRequest>,
+        ) -> Result<tonic::Response<super::GetFeatureResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/GetFeature",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn report_action(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ReportActionRequest>,
+        ) -> Result<tonic::Response<super::ReportActionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/ReportAction",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn report_taxi_action(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ReportTaxiActionRequest>,
+        ) -> Result<tonic::Response<super::ReportTaxiActionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/ReportTaxiAction",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn get_taxi_amount(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetTaxiAmountRequest>,
+        ) -> Result<tonic::Response<super::GetTaxiAmountResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/GetTaxiAmount",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn start_training(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StartTrainingRequest>,
+        ) -> Result<tonic::Response<super::StartTrainingResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/StartTraining",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+    }
+}
+/// Generated server implementations.
+pub mod server_server {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    ///Generated trait containing gRPC methods that should be implemented for use with ServerServer.
+    #[async_trait]
+    pub trait Server: Send + Sync + 'static {
+        async fn get_feature(
+            &self,
+            request: tonic::Request<super::GetFeatureRequest>,
+        ) -> Result<tonic::Response<super::GetFeatureResponse>, tonic::Status>;
+        async fn report_action(
+            &self,
+            request: tonic::Request<super::ReportActionRequest>,
+        ) -> Result<tonic::Response<super::ReportActionResponse>, tonic::Status>;
+        async fn report_taxi_action(
+            &self,
+            request: tonic::Request<super::ReportTaxiActionRequest>,
+        ) -> Result<tonic::Response<super::ReportTaxiActionResponse>, tonic::Status>;
+        async fn get_taxi_amount(
+            &self,
+            request: tonic::Request<super::GetTaxiAmountRequest>,
+        ) -> Result<tonic::Response<super::GetTaxiAmountResponse>, tonic::Status>;
+        async fn start_training(
+            &self,
+            request: tonic::Request<super::StartTrainingRequest>,
+        ) -> Result<tonic::Response<super::StartTrainingResponse>, tonic::Status>;
+    }
+    #[derive(Debug)]
+    pub struct ServerServer<T: Server> {
+        inner: _Inner<T>,
+        accept_compression_encodings: (),
+        send_compression_encodings: (),
+    }
+    struct _Inner<T>(Arc<T>);
+    impl<T: Server> ServerServer<T> {
+        pub fn new(inner: T) -> Self {
+            Self::from_arc(Arc::new(inner))
+        }
+        pub fn from_arc(inner: Arc<T>) -> Self {
+            let inner = _Inner(inner);
+            Self {
+                inner,
+                accept_compression_encodings: Default::default(),
+                send_compression_encodings: Default::default(),
+            }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> InterceptedService<Self, F>
+        where
+            F: tonic::service::Interceptor,
+        {
+            InterceptedService::new(Self::new(inner), interceptor)
+        }
+    }
+    impl<T, B> tonic::codegen::Service<http::Request<B>> for ServerServer<T>
+    where
+        T: Server,
+        B: Body + Send + 'static,
+        B::Error: Into<StdError> + Send + 'static,
+    {
+        type Response = http::Response<tonic::body::BoxBody>;
+        type Error = std::convert::Infallible;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(
+            &mut self,
+            _cx: &mut Context<'_>,
+        ) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, req: http::Request<B>) -> Self::Future {
+            let inner = self.inner.clone();
+            match req.uri().path() {
+                "/server_pb.Server/GetFeature" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetFeatureSvc<T: Server>(pub Arc<T>);
+                    impl<T: Server> tonic::server::UnaryService<super::GetFeatureRequest>
+                    for GetFeatureSvc<T> {
+                        type Response = super::GetFeatureResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::GetFeatureRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move { (*inner).get_feature(request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetFeatureSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/ReportAction" => {
+                    #[allow(non_camel_case_types)]
+                    struct ReportActionSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::ReportActionRequest>
+                    for ReportActionSvc<T> {
+                        type Response = super::ReportActionResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::ReportActionRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).report_action(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = ReportActionSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/ReportTaxiAction" => {
+                    #[allow(non_camel_case_types)]
+                    struct ReportTaxiActionSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::ReportTaxiActionRequest>
+                    for ReportTaxiActionSvc<T> {
+                        type Response = super::ReportTaxiActionResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::ReportTaxiActionRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).report_taxi_action(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = ReportTaxiActionSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/GetTaxiAmount" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetTaxiAmountSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::GetTaxiAmountRequest>
+                    for GetTaxiAmountSvc<T> {
+                        type Response = super::GetTaxiAmountResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::GetTaxiAmountRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).get_taxi_amount(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetTaxiAmountSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/StartTraining" => {
+                    #[allow(non_camel_case_types)]
+                    struct StartTrainingSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::StartTrainingRequest>
+                    for StartTrainingSvc<T> {
+                        type Response = super::StartTrainingResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::StartTrainingRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).start_training(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = StartTrainingSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                _ => {
+                    Box::pin(async move {
+                        Ok(
+                            http::Response::builder()
+                                .status(200)
+                                .header("grpc-status", "12")
+                                .header("content-type", "application/grpc")
+                                .body(empty_body())
+                                .unwrap(),
+                        )
+                    })
+                }
+            }
+        }
+    }
+    impl<T: Server> Clone for ServerServer<T> {
+        fn clone(&self) -> Self {
+            let inner = self.inner.clone();
+            Self {
+                inner,
+                accept_compression_encodings: self.accept_compression_encodings,
+                send_compression_encodings: self.send_compression_encodings,
+            }
+        }
+    }
+    impl<T: Server> Clone for _Inner<T> {
+        fn clone(&self) -> Self {
+            Self(self.0.clone())
+        }
+    }
+    impl<T: std::fmt::Debug> std::fmt::Debug for _Inner<T> {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "{:?}", self.0)
+        }
+    }
+    impl<T: Server> tonic::transport::NamedService for ServerServer<T> {
+        const NAME: &'static str = "server_pb.Server";
+    }
+}
diff --git a/integration_tests/feature-store/server/src/serving.rs b/integration_tests/feature-store/server/src/serving.rs
new file mode 100644
index 0000000000000..c00d8041492c0
--- /dev/null
+++ b/integration_tests/feature-store/server/src/serving.rs
@@ -0,0 +1,67 @@
+use tokio_postgres::NoTls;
+use tonic::{Response, Status};
+
+use crate::model::model_client::ModelClient;
+use crate::model::{GetAmountRequest, TrainingRequest};
+use crate::server_pb::StartTrainingResponse;
+use crate::FeatureStoreServer;
+
+pub const GET_COUNT_SQL: &str = "
+select count from user_mfa_change_count where userid = $1 order by window_start desc limit 1;
+";
+pub const GET_SUM_SQL: &str = "
+select udf_sum from user_mfa_change_sum where userid = $1 order by window_start desc limit 1;
+";
+
+impl FeatureStoreServer {
+    pub async fn get_mfa_feature_from_rw(&self, userid: String) -> Result<(u64, i64), Status> {
+        let con = format!("dbname=dev user=root host=frontend-node-0 port=4566");
+        let (client, connection) = tokio_postgres::connect(&con, NoTls).await.unwrap();
+        tokio::spawn(async move {
+            if let Err(e) = connection.await {
+                eprintln!("connection error: {}", e);
+            }
+        });
+        let data_count = client
+            .query(GET_COUNT_SQL, &[&userid.clone()])
+            .await
+            .unwrap();
+        let data_sum = client.query(GET_SUM_SQL, &[&userid.clone()]).await.unwrap();
+        let count = if data_count.len() == 0 {
+            0 as i64
+        } else {
+            data_count[0].get(0)
+        };
+        let sum = if data_sum.len() == 0 {
+            0 as i64
+        } else {
+            data_sum[0].get(0)
+        };
+        Ok((count as u64, sum))
+    }
+
+    pub async fn do_training(&self) -> Result<Response<StartTrainingResponse>, Status> {
+        let request = TrainingRequest {};
+        let mut model_client = ModelClient::connect("http://localhost:8080")
+            .await
+            .expect("Failed to connect to model server");
+        println!("Training");
+        let response = model_client.training(request).await;
+        match response {
+            Ok(_resp) => Ok(Response::new(StartTrainingResponse {})),
+            Err(e) => Err(e),
+        }
+    }
+
+    pub async fn get_taxi_amount(&self, do_location_id: i64,pu_location_id: i64) -> Result<f32, Status> {
+        let request = GetAmountRequest { do_location_id ,pu_location_id};
+        let mut model_client = ModelClient::connect("http://localhost:8080")
+            .await
+            .expect("Failed to connect to model server");
+        let response = model_client.get_amount(request).await;
+        match response {
+            Ok(resp) => Ok(resp.into_inner().amount),
+            Err(e) => Err(e),
+        }
+    }
+}
diff --git a/integration_tests/feature-store/server/udf.py b/integration_tests/feature-store/server/udf.py
new file mode 100644
index 0000000000000..51651b3534330
--- /dev/null
+++ b/integration_tests/feature-store/server/udf.py
@@ -0,0 +1,23 @@
+import sys
+from typing import Iterator, List, Optional, Tuple, Any
+from decimal import Decimal
+
+sys.path.append("src/udf/python")  # noqa
+
+from risingwave.udf import udf, UdfServer
+
+
+
+@udf(input_types=["INT", "VARCHAR"], result_type="INT")
+def udf_sum(x: int, y: str) -> int:
+    if y=='mfa+':
+        return x
+    else:
+        return -x
+
+
+
+if __name__ == "__main__":
+    server = UdfServer(location="0.0.0.0:8815")
+    server.add_function(udf_sum)
+    server.serve()
\ No newline at end of file
diff --git a/integration_tests/feature-store/simulator/Cargo.lock b/integration_tests/feature-store/simulator/Cargo.lock
new file mode 100644
index 0000000000000..c2be1809ce1bd
--- /dev/null
+++ b/integration_tests/feature-store/simulator/Cargo.lock
@@ -0,0 +1,1506 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"
+
+[[package]]
+name = "async-stream"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "axum"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b9496f0c1d1afb7a2af4338bbe1d969cddfead41d87a9fb3aaa6d0bbc7af648"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bitflags",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "sync_wrapper",
+ "tokio",
+ "tower",
+ "tower-http",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4f44a0e6200e9d11a1cdc989e4b358f6e3d354fbf48478f345a17f4e43f8635"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "mime",
+]
+
+[[package]]
+name = "base64"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bumpalo"
+version = "3.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
+
+[[package]]
+name = "bytes"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
+[[package]]
+name = "csv"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "either"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be"
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "fastrand"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf"
+dependencies = [
+ "instant",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191"
+dependencies = [
+ "matches",
+ "percent-encoding",
+]
+
+[[package]]
+name = "futures"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
+
+[[package]]
+name = "futures-task"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
+
+[[package]]
+name = "futures-util"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "h2"
+version = "0.3.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "http"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "http-range-header"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
+
+[[package]]
+name = "httparse"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c"
+
+[[package]]
+name = "httpdate"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+
+[[package]]
+name = "hyper"
+version = "0.14.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
+[[package]]
+name = "idna"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
+dependencies = [
+ "matches",
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "ipnet"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
+
+[[package]]
+name = "itertools"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+
+[[package]]
+name = "js-sys"
+version = "0.3.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "matches"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
+
+[[package]]
+name = "matchit"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "mime"
+version = "0.3.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
+
+[[package]]
+name = "mio"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
+dependencies = [
+ "libc",
+ "log",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
+
+[[package]]
+name = "openssl"
+version = "0.10.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+
+[[package]]
+name = "pin-project"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "prost"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom 0.2.7",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "reqwest"
+version = "0.11.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92"
+dependencies = [
+ "base64",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "lazy_static",
+ "log",
+ "mime",
+ "native-tls",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+
+[[package]]
+name = "schannel"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
+dependencies = [
+ "lazy_static",
+ "windows-sys",
+]
+
+[[package]]
+name = "security-framework"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.139"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0171ebb889e45aa68b44aee0859b3eede84c6f5f5c228e6f140c0b2a0a46cad6"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.139"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1d3230c1de7932af58ad8ffbe1d784bd55efd5a9d84ac24f69c72d83543dfb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "simulator"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "csv",
+ "futures",
+ "prost",
+ "rand 0.7.3",
+ "reqwest",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tokio",
+ "tonic",
+]
+
+[[package]]
+name = "slab"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
+
+[[package]]
+name = "socket2"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "syn"
+version = "1.0.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
+
+[[package]]
+name = "tempfile"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "libc",
+ "redox_syscall",
+ "remove_dir_all",
+ "winapi",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+
+[[package]]
+name = "tokio"
+version = "1.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57aec3cfa4c296db7255446efb4928a6be304b431a806216105542a67b6ca82e"
+dependencies = [
+ "autocfg",
+ "bytes",
+ "libc",
+ "memchr",
+ "mio",
+ "num_cpus",
+ "once_cell",
+ "pin-project-lite",
+ "socket2",
+ "tokio-macros",
+ "winapi",
+]
+
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-stream"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "tonic"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum",
+ "base64",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "prost-derive",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "indexmap",
+ "pin-project",
+ "pin-project-lite",
+ "rand 0.8.5",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba"
+dependencies = [
+ "bitflags",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-range-header",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
+
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
+[[package]]
+name = "tracing"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160"
+dependencies = [
+ "cfg-if",
+ "log",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "tracing-futures"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
+dependencies = [
+ "pin-project",
+ "tracing",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+
+[[package]]
+name = "url"
+version = "2.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "matches",
+ "percent-encoding",
+]
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "want"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
+dependencies = [
+ "log",
+ "try-lock",
+]
+
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a"
+dependencies = [
+ "bumpalo",
+ "lazy_static",
+ "log",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be"
+
+[[package]]
+name = "web-sys"
+version = "0.3.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+dependencies = [
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+
+[[package]]
+name = "winreg"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
+dependencies = [
+ "winapi",
+]
diff --git a/integration_tests/feature-store/simulator/Cargo.toml b/integration_tests/feature-store/simulator/Cargo.toml
new file mode 100644
index 0000000000000..f0a82f9867ecb
--- /dev/null
+++ b/integration_tests/feature-store/simulator/Cargo.toml
@@ -0,0 +1,22 @@
+[workspace]
+members = []
+
+[package]
+name = "simulator"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+tokio = { version = "1", features=["rt", "rt-multi-thread"]}
+tonic = "0.7.1"
+reqwest = { version = "0.11"}
+serde_json = "1.0"
+serde_derive = "1.0"
+rand = "0.7"
+clap = "2.26.0"
+prost = "0.10"
+serde = { version = "1", features = ["derive"] }
+futures = "0.3.0"
+csv = "1.2.2"
diff --git a/integration_tests/feature-store/simulator/__init__.py b/integration_tests/feature-store/simulator/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/integration_tests/feature-store/simulator/rust-toolchain b/integration_tests/feature-store/simulator/rust-toolchain
new file mode 100644
index 0000000000000..35bda38a1dcfb
--- /dev/null
+++ b/integration_tests/feature-store/simulator/rust-toolchain
@@ -0,0 +1 @@
+nightly-2022-06-20
diff --git a/integration_tests/feature-store/simulator/src/entities.rs b/integration_tests/feature-store/simulator/src/entities.rs
new file mode 100644
index 0000000000000..114a1e5c1ee0f
--- /dev/null
+++ b/integration_tests/feature-store/simulator/src/entities.rs
@@ -0,0 +1,97 @@
+use std::error::Error;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, PathBuf};
+
+use rand::Rng;
+use serde_derive::{Deserialize, Serialize};
+use tonic::transport::Channel;
+
+use crate::server_pb::server_client::ServerClient;
+use crate::server_pb::{GetFeatureRequest, ReportActionRequest};
+
+#[derive(Serialize, Deserialize)]
+pub struct User {
+    pub(crate) userid: String,
+    pub(crate) address_lat: f64,
+    pub(crate) address_long: f64,
+    pub(crate) activeness: f64,
+    pub(crate) age_approx: f64,
+    pub(crate) gender: f64,
+    pub(crate) occupation: f64,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ActionHistory {
+    userid: String,
+    event_type: String,
+    changenum: i64,
+    timestamp: u64,
+}
+
+pub trait UpdatableContext {
+    fn update(&self, record: &ActionHistory);
+}
+
+impl User {
+    pub(crate) async fn mock_act<'a>(
+        &'a self,
+        client: &'a mut ServerClient<Channel>,
+    ) -> Result<ActionHistory, &str> {
+        let changenum: i64 = rand::thread_rng().gen_range(0, 90);
+        let (changenum, event_type) = {
+            if changenum > 0 && changenum < 30 {
+                (changenum, "mfa+")
+            } else if changenum < 60 {
+                (changenum - 30, "mfa-")
+            } else {
+                (0, "other")
+            }
+        };
+        let response = client
+            .report_action(tonic::Request::new(ReportActionRequest {
+                userid: self.userid.clone(),
+                eventtype: event_type.to_string(),
+                changenum,
+            }))
+            .await
+            .unwrap();
+        let timestamp = response.into_inner().timestamp;
+
+        Ok(ActionHistory {
+            userid: self.userid.clone(),
+            changenum: changenum,
+            event_type: event_type.to_string(),
+            timestamp,
+        })
+    }
+
+    #[allow(dead_code)]
+    pub async fn mock_get_feature(&self, client: &mut ServerClient<Channel>) -> (u64, i64) {
+        let response = client
+            .get_feature(GetFeatureRequest {
+                userid: self.userid.clone(),
+            })
+            .await
+            .unwrap();
+
+        let inner = response.into_inner();
+        (inner.count, inner.sum)
+    }
+}
+
+#[allow(dead_code)]
+pub(crate) fn read_users_json(path: PathBuf) -> Result<Vec<User>, Box<dyn Error>> {
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    let users: Vec<User> = serde_json::from_reader(reader)?;
+    Ok(users)
+}
+
+pub fn parse_user_metadata() -> Result<Vec<User>, ()> {
+    let generator_path = std::env::var("GENERATOR_PATH").unwrap_or("../generator".to_string());
+
+    let users = read_users_json(Path::new(&*generator_path).join("users.json")).unwrap();
+
+    return Ok(users);
+}
diff --git a/integration_tests/feature-store/simulator/src/entities_taxi.rs b/integration_tests/feature-store/simulator/src/entities_taxi.rs
new file mode 100644
index 0000000000000..8ef9e1f358e4b
--- /dev/null
+++ b/integration_tests/feature-store/simulator/src/entities_taxi.rs
@@ -0,0 +1,128 @@
+use std::error::Error;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, PathBuf};
+
+use serde_derive::{Deserialize, Serialize};
+use tonic::transport::Channel;
+
+use crate::server_pb::server_client::ServerClient;
+use crate::server_pb::{GetTaxiAmountRequest, ReportTaxiActionRequest};
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct TaxiFeature {
+    #[serde(rename = "vendorID")]
+    pub(crate) vendor_id: i32,
+    #[serde(rename = "lpepPickupDatetime")]
+    pub(crate) lpep_pickup_datetime: String,
+    #[serde(rename = "lpepDropoffDatetime")]
+    pub(crate) lpep_dropoff_datetime: String,
+    #[serde(rename = "storeAndFwdFlag")]
+    pub(crate) store_and_fwd_flag: String,
+    #[serde(rename = "rateCodeID")]
+    pub(crate) ratecode_id: f64,
+    #[serde(rename = "puLocationId")]
+    pub(crate) pulocation_id: i64,
+    #[serde(rename = "doLocationId")]
+    pub(crate) dolocation_id: i64,
+    #[serde(rename = "pickupLongitude")]
+    pub(crate) pickup_longitude: Option<f64>,
+    #[serde(rename = "pickupLatitude")]
+    pub(crate) pickup_latitude: Option<f64>,
+    #[serde(rename = "dropofflongitude")]
+    pub(crate) dropoff_longitude: Option<f64>,
+    #[serde(rename = "dropoffLatitude")]
+    pub(crate) dropoff_latitude: Option<f64>,
+    #[serde(rename = "passengerCount")]
+    pub(crate) passenger_count: f64,
+    #[serde(rename = "tripDistance")]
+    pub(crate) trip_distance: f64,
+    #[serde(rename = "fareAmount")]
+    pub(crate) fare_amount: f64,
+    #[serde(rename = "extra")]
+    pub(crate) extra: f64,
+    #[serde(rename = "mtaTax")]
+    pub(crate) mta_tax: f64,
+    #[serde(rename = "tipAmount")]
+    pub(crate) tip_amount: f64,
+    #[serde(rename = "tollsAmount")]
+    pub(crate) tolls_amount: f64,
+    #[serde(rename = "ehailFee")]
+    pub(crate) ehail_fee: Option<f64>,
+    #[serde(rename = "improvementSurcharge")]
+    pub(crate) improvement_surcharge: f64,
+    #[serde(rename = "totalAmount")]
+    pub(crate) total_amount: f64,
+    #[serde(rename = "paymentType")]
+    pub(crate) payment_type: f64,
+    #[serde(rename = "tripType")]
+    pub(crate) trip_type: f64,
+}
+
+pub fn read_feature_for_csv(path: PathBuf) -> Result<Vec<TaxiFeature>, Box<dyn Error>> {
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    let mut reader = csv::Reader::from_reader(reader);
+    let mut records = vec![];
+    for record in reader.deserialize() {
+        let record: TaxiFeature = record.unwrap();
+        records.push(record);
+    }
+    Ok(records)
+}
+
+pub fn parse_taxi_metadata() -> (Vec<TaxiFeature>, Vec<TaxiFeature>) {
+    let path = std::env::var("WORK_DIR").unwrap_or("/opt/feature-store/".to_string());
+    let mut offlines = read_feature_for_csv(Path::new(&*path).join("parquet_data.csv")).unwrap();
+
+    let onlines = offlines.split_off(offlines.len() / 10 * 9);
+
+    (offlines, onlines)
+}
+
+impl TaxiFeature {
+    pub(crate) async fn mock_act<'a>(
+        &'a self,
+        client: &'a mut ServerClient<Channel>,
+    ) -> Result<(), &str> {
+        let _ = client
+            .report_taxi_action(tonic::Request::new(ReportTaxiActionRequest {
+                vendor_id: self.vendor_id,
+                lpep_pickup_datetime: self.lpep_pickup_datetime.clone(),
+                lpep_dropoff_datetime: self.lpep_dropoff_datetime.clone(),
+                store_and_fwd_flag: self.store_and_fwd_flag.eq("N"),
+                ratecode_id: self.ratecode_id,
+                pu_location_id: self.pulocation_id,
+                do_location_id: self.dolocation_id,
+                passenger_count: self.passenger_count,
+                trip_distance: self.trip_distance,
+                fare_amount: self.fare_amount,
+                extra: self.extra,
+                mta_tax: self.mta_tax,
+                tip_amount: self.tip_amount,
+                tolls_amount: self.tolls_amount,
+                ehail_fee: self.ehail_fee.unwrap_or_else(|| 0.0),
+                improvement_surcharge: self.improvement_surcharge,
+                total_amount: self.total_amount,
+                payment_type: self.payment_type,
+                trip_type: self.trip_type,
+                congestion_surcharge: 0.0,
+            }))
+            .await
+            .unwrap();
+        Ok(())
+    }
+
+    pub async fn mock_get_amount(&self, client: &mut ServerClient<Channel>) -> f64 {
+        let response = client
+            .get_taxi_amount(GetTaxiAmountRequest {
+                do_location_id: self.dolocation_id,
+                pu_location_id: self.pulocation_id,
+            })
+            .await
+            .unwrap();
+
+        let inner = response.into_inner();
+        inner.fare_amount
+    }
+}
diff --git a/integration_tests/feature-store/simulator/src/main.rs b/integration_tests/feature-store/simulator/src/main.rs
new file mode 100644
index 0000000000000..daa6c30af1c74
--- /dev/null
+++ b/integration_tests/feature-store/simulator/src/main.rs
@@ -0,0 +1,31 @@
+use clap::{App, Arg, ArgMatches};
+
+mod entities;
+mod entities_taxi;
+mod server_pb;
+mod simulation;
+
+#[tokio::main]
+async fn main() {
+    let args = get_args();
+    simulation::main_loop(
+        args.value_of("types")
+            .expect("failed to decode brokers")
+            .to_string(),
+    )
+    .await;
+}
+
+fn get_args<'a>() -> ArgMatches<'a> {
+    App::new("simulator")
+        .about("The simulator")
+        .arg(
+            Arg::with_name("types")
+                .short("t")
+                .long("types")
+                .help("mfa or taxi")
+                .takes_value(true)
+                .default_value("taxi"),
+        )
+        .get_matches()
+}
diff --git a/integration_tests/feature-store/simulator/src/server_pb.rs b/integration_tests/feature-store/simulator/src/server_pb.rs
new file mode 100644
index 0000000000000..1f2912d868412
--- /dev/null
+++ b/integration_tests/feature-store/simulator/src/server_pb.rs
@@ -0,0 +1,561 @@
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportActionRequest {
+    #[prost(string, tag="1")]
+    pub userid: ::prost::alloc::string::String,
+    #[prost(string, tag="2")]
+    pub eventtype: ::prost::alloc::string::String,
+    #[prost(int64, tag="3")]
+    pub changenum: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportActionResponse {
+    #[prost(uint64, tag="1")]
+    pub timestamp: u64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetFeatureRequest {
+    #[prost(string, tag="1")]
+    pub userid: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetFeatureResponse {
+    #[prost(uint64, tag="1")]
+    pub count: u64,
+    #[prost(int64, tag="2")]
+    pub sum: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportTaxiActionRequest {
+    #[prost(int32, tag="1")]
+    pub vendor_id: i32,
+    #[prost(string, tag="2")]
+    pub lpep_pickup_datetime: ::prost::alloc::string::String,
+    #[prost(string, tag="3")]
+    pub lpep_dropoff_datetime: ::prost::alloc::string::String,
+    #[prost(bool, tag="4")]
+    pub store_and_fwd_flag: bool,
+    #[prost(double, tag="5")]
+    pub ratecode_id: f64,
+    #[prost(int64, tag="6")]
+    pub pu_location_id: i64,
+    #[prost(int64, tag="7")]
+    pub do_location_id: i64,
+    #[prost(double, tag="8")]
+    pub passenger_count: f64,
+    #[prost(double, tag="9")]
+    pub trip_distance: f64,
+    #[prost(double, tag="10")]
+    pub fare_amount: f64,
+    #[prost(double, tag="11")]
+    pub extra: f64,
+    #[prost(double, tag="12")]
+    pub mta_tax: f64,
+    #[prost(double, tag="13")]
+    pub tip_amount: f64,
+    #[prost(double, tag="14")]
+    pub tolls_amount: f64,
+    #[prost(double, tag="15")]
+    pub ehail_fee: f64,
+    #[prost(double, tag="16")]
+    pub improvement_surcharge: f64,
+    #[prost(double, tag="17")]
+    pub total_amount: f64,
+    #[prost(double, tag="18")]
+    pub payment_type: f64,
+    #[prost(double, tag="19")]
+    pub trip_type: f64,
+    #[prost(double, tag="20")]
+    pub congestion_surcharge: f64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ReportTaxiActionResponse {
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetTaxiAmountRequest {
+    #[prost(int64, tag="1")]
+    pub do_location_id: i64,
+    #[prost(int64, tag="2")]
+    pub pu_location_id: i64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct GetTaxiAmountResponse {
+    #[prost(double, tag="1")]
+    pub fare_amount: f64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct StartTrainingRequest {
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct StartTrainingResponse {
+}
+/// Generated client implementations.
+pub mod server_client {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    #[derive(Debug, Clone)]
+    pub struct ServerClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl ServerClient<tonic::transport::Channel> {
+        /// Attempt to create a new client by connecting to a given endpoint.
+        pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: std::convert::TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+            Ok(Self::new(conn))
+        }
+    }
+    impl<T> ServerClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::BoxBody>,
+        T::Error: Into<StdError>,
+        T::ResponseBody: Body<Data = Bytes> + Send + 'static,
+        <T::ResponseBody as Body>::Error: Into<StdError> + Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> ServerClient<InterceptedService<T, F>>
+        where
+            F: tonic::service::Interceptor,
+            T::ResponseBody: Default,
+            T: tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+                Response = http::Response<
+                    <T as tonic::client::GrpcService<tonic::body::BoxBody>>::ResponseBody,
+                >,
+            >,
+            <T as tonic::codegen::Service<
+                http::Request<tonic::body::BoxBody>,
+            >>::Error: Into<StdError> + Send + Sync,
+        {
+            ServerClient::new(InterceptedService::new(inner, interceptor))
+        }
+        /// Compress requests with `gzip`.
+        ///
+        /// This requires the server to support it otherwise it might respond with an
+        /// error.
+        #[must_use]
+        pub fn send_gzip(mut self) -> Self {
+            self.inner = self.inner.send_gzip();
+            self
+        }
+        /// Enable decompressing responses with `gzip`.
+        #[must_use]
+        pub fn accept_gzip(mut self) -> Self {
+            self.inner = self.inner.accept_gzip();
+            self
+        }
+        pub async fn get_feature(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetFeatureRequest>,
+        ) -> Result<tonic::Response<super::GetFeatureResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/GetFeature",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn report_action(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ReportActionRequest>,
+        ) -> Result<tonic::Response<super::ReportActionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/ReportAction",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn report_taxi_action(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ReportTaxiActionRequest>,
+        ) -> Result<tonic::Response<super::ReportTaxiActionResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/ReportTaxiAction",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn get_taxi_amount(
+            &mut self,
+            request: impl tonic::IntoRequest<super::GetTaxiAmountRequest>,
+        ) -> Result<tonic::Response<super::GetTaxiAmountResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/GetTaxiAmount",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+        pub async fn start_training(
+            &mut self,
+            request: impl tonic::IntoRequest<super::StartTrainingRequest>,
+        ) -> Result<tonic::Response<super::StartTrainingResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/server_pb.Server/StartTraining",
+            );
+            self.inner.unary(request.into_request(), path, codec).await
+        }
+    }
+}
+/// Generated server implementations.
+pub mod server_server {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    ///Generated trait containing gRPC methods that should be implemented for use with ServerServer.
+    #[async_trait]
+    pub trait Server: Send + Sync + 'static {
+        async fn get_feature(
+            &self,
+            request: tonic::Request<super::GetFeatureRequest>,
+        ) -> Result<tonic::Response<super::GetFeatureResponse>, tonic::Status>;
+        async fn report_action(
+            &self,
+            request: tonic::Request<super::ReportActionRequest>,
+        ) -> Result<tonic::Response<super::ReportActionResponse>, tonic::Status>;
+        async fn report_taxi_action(
+            &self,
+            request: tonic::Request<super::ReportTaxiActionRequest>,
+        ) -> Result<tonic::Response<super::ReportTaxiActionResponse>, tonic::Status>;
+        async fn get_taxi_amount(
+            &self,
+            request: tonic::Request<super::GetTaxiAmountRequest>,
+        ) -> Result<tonic::Response<super::GetTaxiAmountResponse>, tonic::Status>;
+        async fn start_training(
+            &self,
+            request: tonic::Request<super::StartTrainingRequest>,
+        ) -> Result<tonic::Response<super::StartTrainingResponse>, tonic::Status>;
+    }
+    #[derive(Debug)]
+    pub struct ServerServer<T: Server> {
+        inner: _Inner<T>,
+        accept_compression_encodings: (),
+        send_compression_encodings: (),
+    }
+    struct _Inner<T>(Arc<T>);
+    impl<T: Server> ServerServer<T> {
+        pub fn new(inner: T) -> Self {
+            Self::from_arc(Arc::new(inner))
+        }
+        pub fn from_arc(inner: Arc<T>) -> Self {
+            let inner = _Inner(inner);
+            Self {
+                inner,
+                accept_compression_encodings: Default::default(),
+                send_compression_encodings: Default::default(),
+            }
+        }
+        pub fn with_interceptor<F>(
+            inner: T,
+            interceptor: F,
+        ) -> InterceptedService<Self, F>
+        where
+            F: tonic::service::Interceptor,
+        {
+            InterceptedService::new(Self::new(inner), interceptor)
+        }
+    }
+    impl<T, B> tonic::codegen::Service<http::Request<B>> for ServerServer<T>
+    where
+        T: Server,
+        B: Body + Send + 'static,
+        B::Error: Into<StdError> + Send + 'static,
+    {
+        type Response = http::Response<tonic::body::BoxBody>;
+        type Error = std::convert::Infallible;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(
+            &mut self,
+            _cx: &mut Context<'_>,
+        ) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, req: http::Request<B>) -> Self::Future {
+            let inner = self.inner.clone();
+            match req.uri().path() {
+                "/server_pb.Server/GetFeature" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetFeatureSvc<T: Server>(pub Arc<T>);
+                    impl<T: Server> tonic::server::UnaryService<super::GetFeatureRequest>
+                    for GetFeatureSvc<T> {
+                        type Response = super::GetFeatureResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::GetFeatureRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move { (*inner).get_feature(request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetFeatureSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/ReportAction" => {
+                    #[allow(non_camel_case_types)]
+                    struct ReportActionSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::ReportActionRequest>
+                    for ReportActionSvc<T> {
+                        type Response = super::ReportActionResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::ReportActionRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).report_action(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = ReportActionSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/ReportTaxiAction" => {
+                    #[allow(non_camel_case_types)]
+                    struct ReportTaxiActionSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::ReportTaxiActionRequest>
+                    for ReportTaxiActionSvc<T> {
+                        type Response = super::ReportTaxiActionResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::ReportTaxiActionRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).report_taxi_action(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = ReportTaxiActionSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/GetTaxiAmount" => {
+                    #[allow(non_camel_case_types)]
+                    struct GetTaxiAmountSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::GetTaxiAmountRequest>
+                    for GetTaxiAmountSvc<T> {
+                        type Response = super::GetTaxiAmountResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::GetTaxiAmountRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).get_taxi_amount(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = GetTaxiAmountSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                "/server_pb.Server/StartTraining" => {
+                    #[allow(non_camel_case_types)]
+                    struct StartTrainingSvc<T: Server>(pub Arc<T>);
+                    impl<
+                        T: Server,
+                    > tonic::server::UnaryService<super::StartTrainingRequest>
+                    for StartTrainingSvc<T> {
+                        type Response = super::StartTrainingResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::StartTrainingRequest>,
+                        ) -> Self::Future {
+                            let inner = self.0.clone();
+                            let fut = async move {
+                                (*inner).start_training(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = StartTrainingSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                _ => {
+                    Box::pin(async move {
+                        Ok(
+                            http::Response::builder()
+                                .status(200)
+                                .header("grpc-status", "12")
+                                .header("content-type", "application/grpc")
+                                .body(empty_body())
+                                .unwrap(),
+                        )
+                    })
+                }
+            }
+        }
+    }
+    impl<T: Server> Clone for ServerServer<T> {
+        fn clone(&self) -> Self {
+            let inner = self.inner.clone();
+            Self {
+                inner,
+                accept_compression_encodings: self.accept_compression_encodings,
+                send_compression_encodings: self.send_compression_encodings,
+            }
+        }
+    }
+    impl<T: Server> Clone for _Inner<T> {
+        fn clone(&self) -> Self {
+            Self(self.0.clone())
+        }
+    }
+    impl<T: std::fmt::Debug> std::fmt::Debug for _Inner<T> {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "{:?}", self.0)
+        }
+    }
+    impl<T: Server> tonic::transport::NamedService for ServerServer<T> {
+        const NAME: &'static str = "server_pb.Server";
+    }
+}
diff --git a/integration_tests/feature-store/simulator/src/simulation.rs b/integration_tests/feature-store/simulator/src/simulation.rs
new file mode 100644
index 0000000000000..96f3c31921a5e
--- /dev/null
+++ b/integration_tests/feature-store/simulator/src/simulation.rs
@@ -0,0 +1,119 @@
+use std::ops::DerefMut;
+use std::sync::Arc;
+use std::thread::sleep;
+use std::time::Duration;
+
+use futures::future::join_all;
+use rand;
+use rand::Rng;
+use tokio::sync::Mutex;
+use tonic::transport::Channel;
+
+use crate::server_pb::server_client::ServerClient;
+use crate::server_pb::StartTrainingRequest;
+use crate::{entities, entities_taxi};
+
+fn get_delay_mills(delay_val: f64) -> u64 {
+    let turbulence =
+        rand::thread_rng().gen_range((delay_val * 0.6) as f64, (delay_val * 1.1) as f64) as f64;
+    (turbulence * 10000.0) as u64
+}
+
+pub async fn main_loop(simulator_type: String) {
+    let client = Arc::new(Mutex::new(
+        ServerClient::connect("https://localhost:2666")
+            .await
+            .expect("failed to connect to feature store server"),
+    ));
+    println!("Connected to server");
+    match simulator_type.as_str() {
+        "taxi" => mock_taxi(client).await,
+        "mfa" => mock_user_mfa(client).await,
+        _ => panic!("Only taxi and mfa"),
+    }
+}
+
+async fn mock_taxi(client: Arc<Mutex<ServerClient<Channel>>>) -> () {
+    let (offline_features, online_features) = entities_taxi::parse_taxi_metadata();
+    println!("Write training data len is {:?}", offline_features.len());
+    let mut threads = vec![];
+    for fea in offline_features {
+        let client_mutex = client.clone();
+        let handle = tokio::spawn(async move {
+            fea.mock_act(client_mutex.lock().await.deref_mut())
+                .await
+                .unwrap();
+        });
+        threads.push(handle);
+    }
+    join_all(threads).await;
+
+    println!("Start training");
+
+    sleep(Duration::from_millis(1000));
+
+    client
+        .lock()
+        .await
+        .deref_mut()
+        .start_training(StartTrainingRequest {})
+        .await
+        .unwrap();
+
+    println!("Offline feature has been written to written in kafka");
+    let mut threads = vec![];
+    for fea in online_features {
+        let client_mutex = client.clone();
+        let handle = tokio::spawn(async move {
+            fea.mock_act(client_mutex.lock().await.deref_mut())
+                .await
+                .unwrap();
+            println!(
+                "write online feature, DOLocationID is {:?}",
+                fea.dolocation_id
+            );
+            tokio::time::sleep(Duration::from_millis(1000)).await;
+            let fare_amount = fea
+                .mock_get_amount(client_mutex.lock().await.deref_mut())
+                .await;
+            println!(
+                "DOLocationID is {:?} fare amount: predicted results {:?} , real results {:?}",
+                fea.dolocation_id, fare_amount, fea.fare_amount
+            );
+        });
+        threads.push(handle);
+    }
+    join_all(threads).await;
+}
+
+#[allow(dead_code)]
+async fn mock_user_mfa(client: Arc<Mutex<ServerClient<Channel>>>) -> () {
+    let users = entities::parse_user_metadata().unwrap();
+    let mut threads = vec![];
+    for user in users {
+        let client_mutex = client.clone();
+        let handle = tokio::spawn(async move {
+            loop {
+                sleep(Duration::from_millis(get_delay_mills(
+                    1.0 / user.activeness,
+                )));
+                let history = user
+                    .mock_act(client_mutex.lock().await.deref_mut())
+                    .await
+                    .unwrap();
+                println!(
+                    "fire action success: {}",
+                    serde_json::to_string(&history).unwrap()
+                );
+
+                sleep(Duration::from_millis(200));
+                let (count, sum) = user
+                    .mock_get_feature(client_mutex.lock().await.deref_mut())
+                    .await;
+                println!("userid {} , count: {:?} sum {:?}", user.userid, count, sum);
+            }
+        });
+        threads.push(handle);
+    }
+    join_all(threads).await;
+}
diff --git a/integration_tests/feature-store/taxi-start.sql b/integration_tests/feature-store/taxi-start.sql
new file mode 100644
index 0000000000000..a74a8685c0b23
--- /dev/null
+++ b/integration_tests/feature-store/taxi-start.sql
@@ -0,0 +1,68 @@
+create source if not exists taxiallfeature (
+    vendor_id int,
+    lpep_pickup_datetime timestamp,
+    lpep_dropoff_datetime timestamp,
+    store_and_fwd_flag boolean,
+    ratecode_id float,
+    pu_location_id int,
+    do_location_id int,
+    passenger_count float,
+    trip_distance float,
+    fare_amount float,
+    extra float,
+    mta_tax float,
+    tip_amount float,
+    tolls_amount float,
+    ehail_fee float,
+    improvement_surcharge float,
+    total_amount float,
+    payment_type float,
+    trip_type float,
+    congestion_surcharge float,
+) with (
+    connector = 'kafka',
+    topic = 'taxi',
+    properties.bootstrap.server = 'kafka:9092',
+)
+FORMAT PLAIN ENCODE JSON;
+
+create materialized view useful_filter as select window_start , lpep_pickup_datetime ,lpep_dropoff_datetime ,do_location_id ,pu_location_id,
+    passenger_count ,trip_distance ,fare_amount ,extra ,mta_tax ,
+    tip_amount,tolls_amount ,improvement_surcharge ,total_amount,
+    congestion_surcharge from (
+    select * from tumble(taxiallfeature,lpep_pickup_datetime,INTERVAL '5' hour)
+) where payment_type in (1,2,4);
+
+create materialized view location_id_store as select do_location_id,pu_location_id,window_start,fare_amount from useful_filter;
+
+create materialized view converted_features_with_do_location as select
+    do_location_id,
+    window_start,
+    avg(EXTRACT(EPOCH FROM lpep_dropoff_datetime - lpep_pickup_datetime)::INT) / 10 as latency,
+    avg(passenger_count) as passenger_count,
+    avg(trip_distance) as trip_distance,
+    avg(extra) as extra,
+    avg(mta_tax) as mta_tax,
+    avg(tip_amount) as tip_amount,
+    avg(tolls_amount) as tolls_amount,
+    avg(improvement_surcharge) as improvement_surcharge,
+    avg(total_amount) as total_amount,
+    avg(congestion_surcharge) as congestion_surcharge,
+    avg(trip_distance) > 30 as long_distance
+from useful_filter group by do_location_id,window_start;
+
+create materialized view converted_features_with_pu_location as select
+    pu_location_id,
+    window_start,
+    avg(EXTRACT(EPOCH FROM lpep_dropoff_datetime - lpep_pickup_datetime)::INT) / 10 as latency,
+    avg(passenger_count) as passenger_count,
+    avg(trip_distance) as trip_distance,
+    avg(extra) as extra,
+    avg(mta_tax) as mta_tax,
+    avg(tip_amount) as tip_amount,
+    avg(tolls_amount) as tolls_amount,
+    avg(improvement_surcharge) as improvement_surcharge,
+    avg(total_amount) as total_amount,
+    avg(congestion_surcharge) as congestion_surcharge,
+    avg(trip_distance) > 30 as long_distance
+from useful_filter group by pu_location_id,window_start;
\ No newline at end of file
diff --git a/integration_tests/mysql-cdc/create_source.sql b/integration_tests/mysql-cdc/create_source.sql
index 9d85863438f8e..62a0d57c699a2 100644
--- a/integration_tests/mysql-cdc/create_source.sql
+++ b/integration_tests/mysql-cdc/create_source.sql
@@ -1,3 +1,5 @@
+set cdc_backfill='true';
+
 CREATE TABLE lineitem_rw (
    L_ORDERKEY BIGINT,
    L_PARTKEY BIGINT,
diff --git a/integration_tests/mysql-sink/create_mv.sql b/integration_tests/mysql-sink/create_mv.sql
index 4a8ef2b7b4709..ba2f9152ce1a8 100644
--- a/integration_tests/mysql-sink/create_mv.sql
+++ b/integration_tests/mysql-sink/create_mv.sql
@@ -31,7 +31,7 @@ CREATE TABLE rw_typed_data (
     boolean_column BOOLEAN,
     date_column DATE,
     time_column TIME,
-    timestamp_column TIMESTAMP,
+    timestamp_column TIMESTAMPTZ,
     jsonb_column JSONB,
     bytea_column BYTEA
 ) WITH (
diff --git a/java/com_risingwave_java_binding_Binding.h b/java/com_risingwave_java_binding_Binding.h
index bd03892223a6d..c2c027ed22b58 100644
--- a/java/com_risingwave_java_binding_Binding.h
+++ b/java/com_risingwave_java_binding_Binding.h
@@ -119,6 +119,70 @@ JNIEXPORT jboolean JNICALL Java_com_risingwave_java_binding_Binding_rowGetBoolea
 JNIEXPORT jstring JNICALL Java_com_risingwave_java_binding_Binding_rowGetStringValue
   (JNIEnv *, jclass, jlong, jint);
 
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetTimestampValue
+ * Signature: (JI)Ljava/sql/Timestamp;
+ */
+JNIEXPORT jobject JNICALL Java_com_risingwave_java_binding_Binding_rowGetTimestampValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetDecimalValue
+ * Signature: (JI)Ljava/math/BigDecimal;
+ */
+JNIEXPORT jobject JNICALL Java_com_risingwave_java_binding_Binding_rowGetDecimalValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetTimeValue
+ * Signature: (JI)Ljava/sql/Time;
+ */
+JNIEXPORT jobject JNICALL Java_com_risingwave_java_binding_Binding_rowGetTimeValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetDateValue
+ * Signature: (JI)Ljava/sql/Date;
+ */
+JNIEXPORT jobject JNICALL Java_com_risingwave_java_binding_Binding_rowGetDateValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetIntervalValue
+ * Signature: (JI)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_com_risingwave_java_binding_Binding_rowGetIntervalValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetJsonbValue
+ * Signature: (JI)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_com_risingwave_java_binding_Binding_rowGetJsonbValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetByteaValue
+ * Signature: (JI)[B
+ */
+JNIEXPORT jbyteArray JNICALL Java_com_risingwave_java_binding_Binding_rowGetByteaValue
+  (JNIEnv *, jclass, jlong, jint);
+
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    rowGetArrayValue
+ * Signature: (JILjava/lang/Class;)Ljava/lang/Object;
+ */
+JNIEXPORT jobject JNICALL Java_com_risingwave_java_binding_Binding_rowGetArrayValue
+  (JNIEnv *, jclass, jlong, jint, jclass);
+
 /*
  * Class:     com_risingwave_java_binding_Binding
  * Method:    rowClose
@@ -151,6 +215,14 @@ JNIEXPORT jlong JNICALL Java_com_risingwave_java_binding_Binding_streamChunkIter
 JNIEXPORT void JNICALL Java_com_risingwave_java_binding_Binding_streamChunkIteratorClose
   (JNIEnv *, jclass, jlong);
 
+/*
+ * Class:     com_risingwave_java_binding_Binding
+ * Method:    streamChunkIteratorFromPretty
+ * Signature: (Ljava/lang/String;)J
+ */
+JNIEXPORT jlong JNICALL Java_com_risingwave_java_binding_Binding_streamChunkIteratorFromPretty
+  (JNIEnv *, jclass, jstring);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/java/connector-node/assembly/assembly.xml b/java/connector-node/assembly/assembly.xml
index 327c7628dbb59..751ba2f410e0b 100644
--- a/java/connector-node/assembly/assembly.xml
+++ b/java/connector-node/assembly/assembly.xml
@@ -40,6 +40,7 @@
 
                 <!-- Sink connectors -->
                 <include>*:risingwave-sink-es-7</include>
+                <include>*:risingwave-sink-cassandra</include>
                 <include>*:risingwave-sink-jdbc</include>
                 <include>*:risingwave-sink-iceberg</include>
                 <include>*:risingwave-sink-deltalake</include>
diff --git a/java/connector-node/assembly/pom.xml b/java/connector-node/assembly/pom.xml
index 5c518601bc8e7..6812bac5b63e6 100644
--- a/java/connector-node/assembly/pom.xml
+++ b/java/connector-node/assembly/pom.xml
@@ -33,6 +33,10 @@
             <groupId>com.risingwave.java</groupId>
             <artifactId>risingwave-sink-es-7</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.risingwave.java</groupId>
+            <artifactId>risingwave-sink-cassandra</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.risingwave.java</groupId>
             <artifactId>risingwave-sink-jdbc</artifactId>
diff --git a/java/connector-node/risingwave-connector-service/pom.xml b/java/connector-node/risingwave-connector-service/pom.xml
index fccd84d24b9af..4e2dbe1d6ec96 100644
--- a/java/connector-node/risingwave-connector-service/pom.xml
+++ b/java/connector-node/risingwave-connector-service/pom.xml
@@ -99,5 +99,10 @@
             <artifactId>risingwave-sink-es-7</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.risingwave.java</groupId>
+            <artifactId>risingwave-sink-cassandra</artifactId>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java
index 6c0f779e287aa..ab3ac84346fa6 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkUtils.java
@@ -43,6 +43,8 @@ public static SinkFactory getSinkFactory(String sinkName) {
                 return new DeltaLakeSinkFactory();
             case "elasticsearch-7":
                 return new EsSink7Factory();
+            case "cassandra":
+                return new CassandraFactory();
             default:
                 throw UNIMPLEMENTED
                         .withDescription("unknown sink type: " + sinkName)
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
index c7756d797196f..5466eca2aaa5b 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
@@ -60,6 +60,8 @@ public class DbzConnectorConfig {
 
     private static final String DBZ_PROPERTY_PREFIX = "debezium.";
 
+    private static final String SNAPSHOT_MODE_BACKFILL = "rw_cdc_backfill";
+
     private static Map<String, String> extractDebeziumProperties(
             Map<String, String> userProperties) {
         // retain only debezium properties if any
@@ -113,12 +115,16 @@ public DbzConnectorConfig(
             // reading from the given offset
             if (snapshotDone && null != startOffset && !startOffset.isBlank()) {
                 // 'snapshot.mode=schema_only_recovery' must be configured if binlog offset is
-                // specified.
-                // It only snapshots the schemas, not the data, and continue binlog reading from the
-                // specified offset
+                // specified. It only snapshots the schemas, not the data, and continue binlog
+                // reading from the specified offset
                 mysqlProps.setProperty("snapshot.mode", "schema_only_recovery");
                 mysqlProps.setProperty(
                         ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
+            } else if (mysqlProps.getProperty("snapshot.mode").equals(SNAPSHOT_MODE_BACKFILL)) {
+                // only snapshot table schemas which is not required by the source parser
+                mysqlProps.setProperty("snapshot.mode", "schema_only");
+                // disable snapshot locking at all
+                mysqlProps.setProperty("snapshot.locking.mode", "none");
             }
 
             dbzProps.putAll(mysqlProps);
@@ -151,7 +157,9 @@ public DbzConnectorConfig(
         }
 
         var otherProps = extractDebeziumProperties(userProps);
-        dbzProps.putAll(otherProps);
+        for (var entry : otherProps.entrySet()) {
+            dbzProps.putIfAbsent(entry.getKey(), entry.getValue());
+        }
 
         this.sourceId = sourceId;
         this.sourceType = source;
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
index e474728ce9c4e..54094bc21862d 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
@@ -147,12 +147,16 @@ private void validateTableSchema() throws SQLException {
 
             // All columns defined must exist in upstream database
             for (var e : tableSchema.getColumnTypes().entrySet()) {
-                var pgDataType = schema.get(e.getKey().toLowerCase());
-                if (pgDataType == null) {
+                // skip validate internal columns
+                if (e.getKey().startsWith(ValidatorUtils.INTERNAL_COLUMN_PREFIX)) {
+                    continue;
+                }
+                var dataType = schema.get(e.getKey().toLowerCase());
+                if (dataType == null) {
                     throw ValidatorUtils.invalidArgument(
                             "Column '" + e.getKey() + "' not found in the upstream database");
                 }
-                if (!isDataTypeCompatible(pgDataType, e.getValue())) {
+                if (!isDataTypeCompatible(dataType, e.getValue())) {
                     throw ValidatorUtils.invalidArgument(
                             "Incompatible data type of column " + e.getKey());
                 }
@@ -166,12 +170,7 @@ private void validateTableSchema() throws SQLException {
 
     private void validatePrivileges() throws SQLException {
         String[] privilegesRequired = {
-            "SELECT",
-            "RELOAD",
-            "SHOW DATABASES",
-            "REPLICATION SLAVE",
-            "REPLICATION CLIENT",
-            "LOCK TABLES"
+            "SELECT", "RELOAD", "SHOW DATABASES", "REPLICATION SLAVE", "REPLICATION CLIENT",
         };
 
         var hashSet = new HashSet<>(List.of(privilegesRequired));
@@ -231,6 +230,8 @@ private boolean isDataTypeCompatible(String mysqlDataType, Data.DataType.TypeNam
                 return val == Data.DataType.TypeName.DECIMAL_VALUE;
             case "varchar":
                 return val == Data.DataType.TypeName.VARCHAR_VALUE;
+            case "timestamp":
+                return val == Data.DataType.TypeName.TIMESTAMPTZ_VALUE;
             default:
                 return true; // true for other uncovered types
         }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
index 68a4df1afefbf..a90ea91de7c01 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
@@ -192,12 +192,16 @@ private void validateTableSchema() throws SQLException {
             }
 
             for (var e : tableSchema.getColumnTypes().entrySet()) {
-                var pgDataType = schema.get(e.getKey().toLowerCase());
-                if (pgDataType == null) {
+                // skip validate internal columns
+                if (e.getKey().startsWith(ValidatorUtils.INTERNAL_COLUMN_PREFIX)) {
+                    continue;
+                }
+                var dataType = schema.get(e.getKey().toLowerCase());
+                if (dataType == null) {
                     throw ValidatorUtils.invalidArgument(
                             "Column '" + e.getKey() + "' not found in the upstream database");
                 }
-                if (!isDataTypeCompatible(pgDataType, e.getValue())) {
+                if (!isDataTypeCompatible(dataType, e.getValue())) {
                     throw ValidatorUtils.invalidArgument(
                             "Incompatible data type of column " + e.getKey());
                 }
@@ -262,11 +266,15 @@ private void validatePrivileges() throws SQLException {
         }
 
         // check whether select privilege on table for snapshot read
-        validateTablePrivileges();
+        validateTablePrivileges(isSuperUser);
         validatePublicationConfig(isSuperUser);
     }
 
-    private void validateTablePrivileges() throws SQLException {
+    private void validateTablePrivileges(boolean isSuperUser) throws SQLException {
+        if (isSuperUser) {
+            return;
+        }
+
         try (var stmt =
                 jdbcConnection.prepareStatement(
                         ValidatorUtils.getSql("postgres.table_read_privilege.check"))) {
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java
index 3634346268566..710cde324cd99 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java
@@ -27,6 +27,7 @@ public final class ValidatorUtils {
     static final Logger LOG = LoggerFactory.getLogger(ValidatorUtils.class);
 
     static final String VALIDATE_SQL_FILE = "validate_sql.properties";
+    static final String INTERNAL_COLUMN_PREFIX = "_rw_";
 
     public static RuntimeException failedPrecondition(String description) {
         return Status.FAILED_PRECONDITION.withDescription(description).asRuntimeException();
diff --git a/java/connector-node/risingwave-connector-test/pom.xml b/java/connector-node/risingwave-connector-test/pom.xml
index d630eb183c856..7f8f6f1bc49cc 100644
--- a/java/connector-node/risingwave-connector-test/pom.xml
+++ b/java/connector-node/risingwave-connector-test/pom.xml
@@ -174,5 +174,10 @@
             <artifactId>risingwave-sink-es-7</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.risingwave.java</groupId>
+            <artifactId>risingwave-sink-cassandra</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/source/MySQLSourceTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/source/MySQLSourceTest.java
index 5c1e78f3dd086..294ebffcc70e8 100644
--- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/source/MySQLSourceTest.java
+++ b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/source/MySQLSourceTest.java
@@ -193,9 +193,7 @@ public void testPermissionCheck() throws SQLException {
                             tableSchema,
                             "test",
                             "orders");
-            assertEquals(
-                    "INVALID_ARGUMENT: MySQL user doesn't have enough privileges: [LOCK TABLES]",
-                    resp.getError().getErrorMessage());
+            assertEquals("", resp.getError().getErrorMessage());
         } catch (Exception e) {
             Assert.fail("validate rpc fail: " + e.getMessage());
         } finally {
diff --git a/java/connector-node/risingwave-sink-cassandra/pom.xml b/java/connector-node/risingwave-sink-cassandra/pom.xml
new file mode 100644
index 0000000000000..e51faa9691cf9
--- /dev/null
+++ b/java/connector-node/risingwave-sink-cassandra/pom.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>java-parent</artifactId>
+        <groupId>com.risingwave.java</groupId>
+        <version>1.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>risingwave-sink-cassandra</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <name>risingwave-sink-cassandra</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.risingwave.java</groupId>
+            <artifactId>proto</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.risingwave.java</groupId>
+            <artifactId>connector-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <!-- cassandra drivers -->
+        <dependency>
+            <groupId>com.datastax.oss</groupId>
+            <artifactId>java-driver-core</artifactId>
+            <version>${datastax.version}</version>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java
new file mode 100644
index 0000000000000..a993a8988e1ee
--- /dev/null
+++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2023 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.risingwave.connector;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.risingwave.connector.api.sink.CommonSinkConfig;
+
+public class CassandraConfig extends CommonSinkConfig {
+    /** Required */
+    private String type;
+    /** Required */
+    private String url;
+
+    /** Required */
+    private String keyspace;
+
+    /** Required */
+    private String table;
+
+    /** Required */
+    private String datacenter;
+
+    @JsonProperty(value = "cassandra.username")
+    private String username;
+
+    @JsonProperty(value = "cassandra.password")
+    private String password;
+
+    @JsonCreator
+    public CassandraConfig(
+            @JsonProperty(value = "cassandra.url") String url,
+            @JsonProperty(value = "cassandra.keyspace") String keyspace,
+            @JsonProperty(value = "cassandra.table") String table,
+            @JsonProperty(value = "cassandra.datacenter") String datacenter,
+            @JsonProperty(value = "type") String type) {
+        this.url = url;
+        this.keyspace = keyspace;
+        this.table = table;
+        this.datacenter = datacenter;
+        this.type = type;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getKeyspace() {
+        return keyspace;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public String getDatacenter() {
+        return datacenter;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public CassandraConfig withUsername(String username) {
+        this.username = username;
+        return this;
+    }
+
+    public CassandraConfig withPassword(String password) {
+        this.password = password;
+        return this;
+    }
+}
diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraFactory.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraFactory.java
new file mode 100644
index 0000000000000..f9fb5bee020e2
--- /dev/null
+++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraFactory.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2023 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.risingwave.connector;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import com.datastax.oss.driver.api.core.CqlSessionBuilder;
+import com.datastax.oss.driver.api.core.cql.*;
+import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.risingwave.connector.api.TableSchema;
+import com.risingwave.connector.api.sink.SinkFactory;
+import com.risingwave.connector.api.sink.SinkWriter;
+import com.risingwave.connector.api.sink.SinkWriterV1;
+import com.risingwave.proto.Catalog.SinkType;
+import io.grpc.Status;
+import java.net.InetSocketAddress;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CassandraFactory implements SinkFactory {
+    private static final Logger LOG = LoggerFactory.getLogger(CassandraFactory.class);
+
+    public SinkWriter createWriter(TableSchema tableSchema, Map<String, String> tableProperties) {
+        ObjectMapper mapper = new ObjectMapper();
+        CassandraConfig config = mapper.convertValue(tableProperties, CassandraConfig.class);
+        return new SinkWriterV1.Adapter(new CassandraSink(tableSchema, config));
+    }
+
+    @Override
+    public void validate(
+            TableSchema tableSchema, Map<String, String> tableProperties, SinkType sinkType) {
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.configure(DeserializationFeature.FAIL_ON_MISSING_CREATOR_PROPERTIES, true);
+        CassandraConfig config = mapper.convertValue(tableProperties, CassandraConfig.class);
+
+        // 1. check url
+        String url = config.getUrl();
+        String[] hostPort = url.split(":");
+        if (hostPort.length != 2) {
+            throw new IllegalArgumentException(
+                    "Invalid cassandraURL: expected `host:port`, got " + url);
+        }
+        // 2. check connection
+        CqlSessionBuilder sessionBuilder =
+                CqlSession.builder()
+                        .addContactPoint(
+                                new InetSocketAddress(hostPort[0], Integer.parseInt(hostPort[1])))
+                        .withKeyspace(config.getKeyspace())
+                        .withLocalDatacenter(config.getDatacenter());
+        if (config.getUsername() != null && config.getPassword() != null) {
+            sessionBuilder =
+                    sessionBuilder.withAuthCredentials(config.getUsername(), config.getPassword());
+        }
+        CqlSession session = sessionBuilder.build();
+
+        TableMetadata tableMetadata =
+                session.getMetadata()
+                        .getKeyspace(config.getKeyspace())
+                        .get()
+                        .getTable(config.getTable())
+                        .get();
+        CassandraUtil.checkSchema(tableSchema.getColumnDescs(), tableMetadata.getColumns());
+
+        if (session.isClosed()) {
+            throw Status.INVALID_ARGUMENT
+                    .withDescription("Cannot connect to " + config.getUrl())
+                    .asRuntimeException();
+        }
+        // 3. close client
+        session.close();
+        switch (sinkType) {
+            case UPSERT:
+                CassandraUtil.checkPrimaryKey(
+                        tableMetadata.getPrimaryKey(), tableSchema.getPrimaryKeys());
+                break;
+            case APPEND_ONLY:
+            case FORCE_APPEND_ONLY:
+                break;
+            default:
+                throw Status.INTERNAL.asRuntimeException();
+        }
+    }
+}
diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java
new file mode 100644
index 0000000000000..af422dd2f5bdb
--- /dev/null
+++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2023 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.risingwave.connector;
+
+import com.datastax.oss.driver.api.core.CqlSession;
+import com.datastax.oss.driver.api.core.CqlSessionBuilder;
+import com.datastax.oss.driver.api.core.cql.*;
+import com.risingwave.connector.api.TableSchema;
+import com.risingwave.connector.api.sink.SinkRow;
+import com.risingwave.connector.api.sink.SinkWriterBase;
+import com.risingwave.proto.Data;
+import com.risingwave.proto.Data.DataType.TypeName;
+import io.grpc.Status;
+import java.net.InetSocketAddress;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CassandraSink extends SinkWriterBase {
+    private static final Logger LOG = LoggerFactory.getLogger(CassandraSink.class);
+    private final CqlSession session;
+    private final List<SinkRow> updateRowCache = new ArrayList<>(1);
+    private final HashMap<String, PreparedStatement> stmtMap;
+    private final List<String> nonKeyColumns;
+    private final BatchStatementBuilder batchBuilder;
+    private final CassandraConfig config;
+
+    public CassandraSink(TableSchema tableSchema, CassandraConfig config) {
+        super(tableSchema);
+        String url = config.getUrl();
+        String[] hostPort = url.split(":");
+        if (hostPort.length != 2) {
+            throw new IllegalArgumentException(
+                    "Invalid cassandraURL: expected `host:port`, got " + url);
+        }
+        // check connection
+        CqlSessionBuilder sessionBuilder =
+                CqlSession.builder()
+                        .addContactPoint(
+                                new InetSocketAddress(hostPort[0], Integer.parseInt(hostPort[1])))
+                        .withKeyspace(config.getKeyspace())
+                        .withLocalDatacenter(config.getDatacenter());
+        if (config.getUsername() != null && config.getPassword() != null) {
+            sessionBuilder =
+                    sessionBuilder.withAuthCredentials(config.getUsername(), config.getPassword());
+        }
+        this.session = sessionBuilder.build();
+        if (session.isClosed()) {
+            throw Status.INVALID_ARGUMENT
+                    .withDescription("Cannot connect to " + config.getUrl())
+                    .asRuntimeException();
+        }
+
+        this.config = config;
+        this.batchBuilder = BatchStatement.builder(DefaultBatchType.LOGGED);
+
+        // fetch non-pk columns for prepared statements
+        nonKeyColumns =
+                Arrays.stream(tableSchema.getColumnNames())
+                        // cassandra does not allow SET on primary keys
+                        .filter(c -> !tableSchema.getPrimaryKeys().contains(c))
+                        .collect(Collectors.toList());
+
+        this.stmtMap = new HashMap<>();
+        // prepare statement for insert
+        this.stmtMap.put(
+                "insert", session.prepare(createInsertStatement(config.getTable(), tableSchema)));
+        if (config.getType().equals("upsert")) {
+            // prepare statement for update-insert/update-delete
+            this.stmtMap.put(
+                    "update",
+                    session.prepare(createUpdateStatement(config.getTable(), tableSchema)));
+
+            // prepare the delete statement
+            this.stmtMap.put(
+                    "delete",
+                    session.prepare(createDeleteStatement(config.getTable(), tableSchema)));
+        }
+    }
+
+    @Override
+    public void write(Iterator<SinkRow> rows) {
+        if (this.config.getType().equals("append-only")) {
+            write_append_only(rows);
+        } else {
+            write_upsert(rows);
+        }
+    }
+
+    private void write_append_only(Iterator<SinkRow> rows) {
+        while (rows.hasNext()) {
+            SinkRow row = rows.next();
+            Data.Op op = row.getOp();
+            switch (op) {
+                case INSERT:
+                    batchBuilder.addStatement(bindInsertStatement(this.stmtMap.get("insert"), row));
+                    break;
+                case UPDATE_DELETE:
+                    break;
+                case UPDATE_INSERT:
+                    break;
+                case DELETE:
+                    break;
+                default:
+                    throw Status.INTERNAL
+                            .withDescription("Unknown operation: " + op)
+                            .asRuntimeException();
+            }
+        }
+    }
+
+    private void write_upsert(Iterator<SinkRow> rows) {
+        while (rows.hasNext()) {
+            SinkRow row = rows.next();
+            Data.Op op = row.getOp();
+            switch (op) {
+                case INSERT:
+                    batchBuilder.addStatement(bindInsertStatement(this.stmtMap.get("insert"), row));
+                    break;
+                case UPDATE_DELETE:
+                    updateRowCache.clear();
+                    updateRowCache.add(row);
+                    break;
+                case UPDATE_INSERT:
+                    SinkRow old = updateRowCache.remove(0);
+                    if (old == null) {
+                        throw Status.FAILED_PRECONDITION
+                                .withDescription("UPDATE_INSERT without UPDATE_DELETE")
+                                .asRuntimeException();
+                    }
+                    batchBuilder.addStatement(
+                            bindUpdateInsertStatement(this.stmtMap.get("update"), old, row));
+                    break;
+                case DELETE:
+                    batchBuilder.addStatement(bindDeleteStatement(this.stmtMap.get("delete"), row));
+                    break;
+                default:
+                    throw Status.INTERNAL
+                            .withDescription("Unknown operation: " + op)
+                            .asRuntimeException();
+            }
+        }
+    }
+
+    @Override
+    public void sync() {
+        try {
+            session.execute(batchBuilder.build());
+            batchBuilder.clearStatements();
+        } catch (Exception e) {
+            throw Status.INTERNAL.withDescription(e.getMessage()).asRuntimeException();
+        }
+    }
+
+    @Override
+    public void drop() {
+        session.close();
+    }
+
+    private String createInsertStatement(String tableName, TableSchema tableSchema) {
+        String[] columnNames = tableSchema.getColumnNames();
+        String columnNamesString = String.join(", ", columnNames);
+        String placeholdersString = String.join(", ", Collections.nCopies(columnNames.length, "?"));
+        return String.format(
+                "INSERT INTO %s (%s) VALUES (%s)",
+                tableName, columnNamesString, placeholdersString);
+    }
+
+    private String createUpdateStatement(String tableName, TableSchema tableSchema) {
+        List<String> primaryKeys = tableSchema.getPrimaryKeys();
+        String setClause = // cassandra does not allow SET on primary keys
+                nonKeyColumns.stream()
+                        .map(columnName -> columnName + " = ?")
+                        .collect(Collectors.joining(", "));
+        String whereClause =
+                primaryKeys.stream()
+                        .map(columnName -> columnName + " = ?")
+                        .collect(Collectors.joining(" AND "));
+        return String.format("UPDATE %s SET %s WHERE %s", tableName, setClause, whereClause);
+    }
+
+    private static String createDeleteStatement(String tableName, TableSchema tableSchema) {
+        List<String> primaryKeys = tableSchema.getPrimaryKeys();
+        String whereClause =
+                primaryKeys.stream()
+                        .map(columnName -> columnName + " = ?")
+                        .collect(Collectors.joining(" AND "));
+        return String.format("DELETE FROM %s WHERE %s", tableName, whereClause);
+    }
+
+    private BoundStatement bindInsertStatement(PreparedStatement stmt, SinkRow row) {
+        TableSchema schema = getTableSchema();
+        return stmt.bind(
+                IntStream.range(0, row.size())
+                        .mapToObj(
+                                (index) ->
+                                        CassandraUtil.convertRow(
+                                                row.get(index),
+                                                schema.getColumnDescs()
+                                                        .get(index)
+                                                        .getDataType()
+                                                        .getTypeName()))
+                        .toArray());
+    }
+
+    private BoundStatement bindDeleteStatement(PreparedStatement stmt, SinkRow row) {
+        TableSchema schema = getTableSchema();
+        Map<String, TypeName> columnDescs = schema.getColumnTypes();
+        return stmt.bind(
+                getTableSchema().getPrimaryKeys().stream()
+                        .map(
+                                key ->
+                                        CassandraUtil.convertRow(
+                                                schema.getFromRow(key, row), columnDescs.get(key)))
+                        .toArray());
+    }
+
+    private BoundStatement bindUpdateInsertStatement(
+            PreparedStatement stmt, SinkRow updateRow, SinkRow insertRow) {
+        TableSchema schema = getTableSchema();
+        int numKeys = schema.getPrimaryKeys().size();
+        int numNonKeys = updateRow.size() - numKeys;
+        Map<String, TypeName> columnDescs = schema.getColumnTypes();
+        Object[] values = new Object[numNonKeys + numKeys];
+
+        // bind "SET" clause
+        Iterator<String> nonKeyIter = nonKeyColumns.iterator();
+        for (int i = 0; i < numNonKeys; i++) {
+            String name = nonKeyIter.next();
+            values[i] =
+                    CassandraUtil.convertRow(
+                            schema.getFromRow(name, insertRow), columnDescs.get(name));
+        }
+
+        // bind "WHERE" clause
+        Iterator<String> keyIter = schema.getPrimaryKeys().iterator();
+        for (int i = 0; i < numKeys; i++) {
+            String name = keyIter.next();
+            values[numNonKeys + i] =
+                    CassandraUtil.convertRow(
+                            schema.getFromRow(name, updateRow), columnDescs.get(name));
+        }
+        return stmt.bind(values);
+    }
+}
diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java
new file mode 100644
index 0000000000000..fbe9c1c04a28b
--- /dev/null
+++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2023 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.risingwave.connector;
+
+import com.datastax.oss.driver.api.core.CqlIdentifier;
+import com.datastax.oss.driver.api.core.data.CqlDuration;
+import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata;
+import com.risingwave.connector.api.ColumnDesc;
+import com.risingwave.proto.Data.DataType;
+import com.risingwave.proto.Data.DataType.TypeName;
+import io.grpc.Status;
+import java.nio.ByteBuffer;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class CassandraUtil {
+    private static int getCorrespondingCassandraType(DataType dataType) {
+        switch (dataType.getTypeName()) {
+            case INT16:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.SMALLINT;
+            case INT32:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.INT;
+            case INT64:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.BIGINT;
+            case FLOAT:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.FLOAT;
+            case DOUBLE:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.DOUBLE;
+            case BOOLEAN:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.BOOLEAN;
+            case VARCHAR:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.VARCHAR;
+            case DECIMAL:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.DECIMAL;
+            case TIMESTAMP:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.TIMESTAMP;
+            case TIMESTAMPTZ:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.TIMESTAMP;
+            case DATE:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.DATE;
+            case TIME:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.TIME;
+            case BYTEA:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.BLOB;
+            case LIST:
+            case STRUCT:
+                throw Status.UNIMPLEMENTED
+                        .withDescription(String.format("not support %s now", dataType))
+                        .asRuntimeException();
+            case INTERVAL:
+                return com.datastax.oss.protocol.internal.ProtocolConstants.DataType.DURATION;
+            default:
+                throw Status.INVALID_ARGUMENT
+                        .withDescription("unspecified type" + dataType)
+                        .asRuntimeException();
+        }
+    }
+
+    public static void checkSchema(
+            List<ColumnDesc> columnDescs,
+            Map<CqlIdentifier, ColumnMetadata> cassandraColumnDescMap) {
+        if (columnDescs.size() != cassandraColumnDescMap.size()) {
+            throw Status.FAILED_PRECONDITION
+                    .withDescription("Don't match in the number of columns in the table")
+                    .asRuntimeException();
+        }
+        for (ColumnDesc columnDesc : columnDescs) {
+            CqlIdentifier cql = CqlIdentifier.fromInternal(columnDesc.getName());
+            if (!cassandraColumnDescMap.containsKey(cql)) {
+                throw Status.FAILED_PRECONDITION
+                        .withDescription(
+                                String.format(
+                                        "Don't match in the name, rw is %s cassandra can't find it",
+                                        columnDesc.getName()))
+                        .asRuntimeException();
+            }
+            if (cassandraColumnDescMap.get(cql).getType().getProtocolCode()
+                    != getCorrespondingCassandraType(columnDesc.getDataType())) {
+                throw Status.FAILED_PRECONDITION
+                        .withDescription(
+                                String.format(
+                                        "Don't match in the type, name is %s, cassandra is %s, rw is %s",
+                                        columnDesc.getName(),
+                                        cassandraColumnDescMap.get(cql),
+                                        columnDesc.getDataType().getTypeName()))
+                        .asRuntimeException();
+            }
+        }
+    }
+
+    public static void checkPrimaryKey(
+            List<ColumnMetadata> cassandraColumnMetadatas, List<String> columnMetadatas) {
+        if (cassandraColumnMetadatas.size() != columnMetadatas.size()) {
+            throw Status.FAILED_PRECONDITION
+                    .withDescription("Primary key len don't match")
+                    .asRuntimeException();
+        }
+        Set<String> cassandraColumnsSet =
+                cassandraColumnMetadatas.stream()
+                        .map((a) -> a.getName().toString())
+                        .collect(Collectors.toSet());
+        for (String columnMetadata : columnMetadatas) {
+            if (!cassandraColumnsSet.contains(columnMetadata)) {
+                throw Status.FAILED_PRECONDITION
+                        .withDescription(
+                                String.format(
+                                        "Primary key don't match. RisingWave Primary key is %s, don't find it in cassandra",
+                                        columnMetadata))
+                        .asRuntimeException();
+            }
+        }
+    }
+
+    public static Object convertRow(Object value, TypeName typeName) {
+        switch (typeName) {
+            case INT16:
+            case INT32:
+            case INT64:
+            case FLOAT:
+            case DOUBLE:
+            case BOOLEAN:
+            case VARCHAR:
+            case DECIMAL:
+                return value;
+            case TIMESTAMP:
+            case TIMESTAMPTZ:
+                return ((Timestamp) value).toInstant();
+            case DATE:
+                return ((Date) value).toLocalDate();
+            case TIME:
+                return ((Time) value).toLocalTime();
+            case INTERVAL:
+                return CqlDuration.from((String) value);
+            case BYTEA:
+                return ByteBuffer.wrap((byte[]) value);
+            case LIST:
+            case STRUCT:
+                throw Status.UNIMPLEMENTED
+                        .withDescription(String.format("not support %s now", typeName))
+                        .asRuntimeException();
+            default:
+                throw Status.INVALID_ARGUMENT
+                        .withDescription("unspecified type" + typeName)
+                        .asRuntimeException();
+        }
+    }
+}
diff --git a/java/connector-node/risingwave-sink-es-7/pom.xml b/java/connector-node/risingwave-sink-es-7/pom.xml
index dfeef31e83b90..04694ed98bb04 100644
--- a/java/connector-node/risingwave-sink-es-7/pom.xml
+++ b/java/connector-node/risingwave-sink-es-7/pom.xml
@@ -9,7 +9,6 @@
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-
     <artifactId>risingwave-sink-es-7</artifactId>
     <version>1.0-SNAPSHOT</version>
     <name>risingwave-sink-es-7</name>
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java
index 8d1d0493d6465..b93c3d2ab3824 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java
@@ -45,7 +45,8 @@ public String getNormalizedTableName(SchemaTableName schemaTableName) {
 
     @Override
     public String quoteIdentifier(String identifier) {
-        return identifier;
+        // quote identifier will be case-sensitive in postgres
+        return "\"" + identifier + "\"";
     }
 
     @Override
diff --git a/java/pom.xml b/java/pom.xml
index 401db2aed123a..e7c790e37a79f 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -18,6 +18,7 @@
         <module>connector-node/risingwave-sink-iceberg</module>
         <module>connector-node/risingwave-sink-deltalake</module>
         <module>connector-node/risingwave-sink-es-7</module>
+        <module>connector-node/risingwave-sink-cassandra</module>
         <module>connector-node/risingwave-sink-jdbc</module>
         <module>connector-node/risingwave-source-cdc</module>
         <module>connector-node/risingwave-connector-test</module>
@@ -45,6 +46,7 @@
         <spark_sql.version>3.3.1</spark_sql.version>
         <hadoop.version>3.3.3</hadoop.version>
         <elasticsearch.version>7.17.10</elasticsearch.version>
+        <datastax.version>4.15.0</datastax.version>
     </properties>
 
     <dependencyManagement>
@@ -225,6 +227,11 @@
                 <artifactId>risingwave-sink-es-7</artifactId>
                 <version>${module.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.risingwave.java</groupId>
+                <artifactId>risingwave-sink-cassandra</artifactId>
+                <version>${module.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.risingwave.java</groupId>
                 <artifactId>risingwave-sink-jdbc</artifactId>
diff --git a/proto/catalog.proto b/proto/catalog.proto
index 85eb266ca0527..aa09274a0d818 100644
--- a/proto/catalog.proto
+++ b/proto/catalog.proto
@@ -45,6 +45,7 @@ message StreamSourceInfo {
   plan_common.EncodeType row_encode = 9;
   SchemaRegistryNameStrategy name_strategy = 10;
   optional string key_message_name = 11;
+  plan_common.StorageTableDesc upstream_table = 12;
 }
 
 message Source {
@@ -113,6 +114,8 @@ message Sink {
   optional uint32 connection_id = 14;
   optional uint64 initialized_at_epoch = 15;
   optional uint64 created_at_epoch = 16;
+  string db_name = 17;
+  string sink_from_name = 18;
 }
 
 message Connection {
diff --git a/proto/connector_service.proto b/proto/connector_service.proto
index 3704897ecd595..54728bf4ecdec 100644
--- a/proto/connector_service.proto
+++ b/proto/connector_service.proto
@@ -24,6 +24,8 @@ message SinkParam {
   map<string, string> properties = 2;
   TableSchema table_schema = 3;
   catalog.SinkType sink_type = 4;
+  string db_name = 5;
+  string sink_from_name = 6;
 }
 
 enum SinkPayloadFormat {
diff --git a/proto/expr.proto b/proto/expr.proto
index a7fdc9f3efbb4..9bd5af893cb8b 100644
--- a/proto/expr.proto
+++ b/proto/expr.proto
@@ -102,6 +102,8 @@ message ExprNode {
     BIT_LENGTH = 230;
     OVERLAY = 231;
     REGEXP_MATCH = 232;
+    REGEXP_REPLACE = 280;
+    REGEXP_COUNT = 281;
     POW = 233;
     EXP = 234;
     CHR = 235;
@@ -194,6 +196,9 @@ message ExprNode {
     ARRAY_POSITION = 542;
     ARRAY_REPLACE = 543;
     ARRAY_DIMS = 544;
+    ARRAY_TRANSFORM = 545;
+    ARRAY_MIN = 546;
+    ARRAY_MAX = 547;
 
     // Int256 functions
     HEX_TO_INT256 = 560;
diff --git a/proto/hummock.proto b/proto/hummock.proto
index cb5afa03de5f1..b0a5152821009 100644
--- a/proto/hummock.proto
+++ b/proto/hummock.proto
@@ -374,11 +374,6 @@ message CompactTaskProgress {
   uint64 num_pending_write_io = 6;
 }
 
-// The measurement of the workload on a compactor to determine whether it is idle.
-message CompactorWorkload {
-  uint32 cpu = 1;
-}
-
 message SubscribeCompactionEventRequest {
   // Register provides the context_id of the corresponding Compactor.
   message Register {
diff --git a/proto/meta.proto b/proto/meta.proto
index b5f3b7ce7ec5f..53cf83a28fb2d 100644
--- a/proto/meta.proto
+++ b/proto/meta.proto
@@ -130,6 +130,30 @@ message FlushResponse {
   hummock.HummockSnapshot snapshot = 2;
 }
 
+// The reason why the data sources in the cluster are paused.
+enum PausedReason {
+  PAUSED_REASON_UNSPECIFIED = 0;
+  // The cluster is paused due to configuration change, e.g. altering table schema and scaling.
+  PAUSED_REASON_CONFIG_CHANGE = 1;
+  // The cluster is paused due to manual operation, e.g. `risectl` command or the
+  // `pause_on_next_bootstrap` system variable.
+  PAUSED_REASON_MANUAL = 2;
+}
+
+message PauseRequest {}
+
+message PauseResponse {
+  optional PausedReason prev = 1;
+  optional PausedReason curr = 2;
+}
+
+message ResumeRequest {}
+
+message ResumeResponse {
+  optional PausedReason prev = 1;
+  optional PausedReason curr = 2;
+}
+
 message CancelCreatingJobsRequest {
   message CreatingJobInfo {
     uint32 database_id = 1;
@@ -215,6 +239,8 @@ message ListActorStatesResponse {
 
 service StreamManagerService {
   rpc Flush(FlushRequest) returns (FlushResponse);
+  rpc Pause(PauseRequest) returns (PauseResponse);
+  rpc Resume(ResumeRequest) returns (ResumeResponse);
   rpc CancelCreatingJobs(CancelCreatingJobsRequest) returns (CancelCreatingJobsResponse);
   rpc ListTableFragments(ListTableFragmentsRequest) returns (ListTableFragmentsResponse);
   rpc ListTableFragmentStates(ListTableFragmentStatesRequest) returns (ListTableFragmentStatesResponse);
@@ -389,14 +415,6 @@ service NotificationService {
   rpc Subscribe(SubscribeRequest) returns (stream SubscribeResponse);
 }
 
-message PauseRequest {}
-
-message PauseResponse {}
-
-message ResumeRequest {}
-
-message ResumeResponse {}
-
 message GetClusterInfoRequest {}
 
 message GetClusterInfoResponse {
@@ -431,6 +449,7 @@ message GetReschedulePlanRequest {
     repeated uint32 include_worker_ids = 1;
     repeated uint32 exclude_worker_ids = 2;
     optional uint32 target_parallelism = 3;
+    optional uint32 target_parallelism_per_worker = 4;
   }
 
   message StableResizePolicy {
@@ -454,9 +473,6 @@ message GetReschedulePlanResponse {
 }
 
 service ScaleService {
-  // TODO(Kexiang): delete them when config change interface is finished
-  rpc Pause(PauseRequest) returns (PauseResponse);
-  rpc Resume(ResumeRequest) returns (ResumeResponse);
   rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse);
   rpc Reschedule(RescheduleRequest) returns (RescheduleResponse);
   rpc GetReschedulePlan(GetReschedulePlanRequest) returns (GetReschedulePlanResponse);
@@ -495,6 +511,7 @@ message SystemParams {
   optional bool telemetry_enabled = 10;
   optional uint32 parallel_compact_size_mb = 11;
   optional uint32 max_concurrent_creating_streaming_jobs = 12;
+  optional bool pause_on_next_bootstrap = 13;
 }
 
 message GetSystemParamsRequest {}
diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto
index 92a5dc89ee70d..33fe96a71c803 100644
--- a/proto/stream_plan.proto
+++ b/proto/stream_plan.proto
@@ -24,6 +24,9 @@ message AddMutation {
   // We may embed a source change split mutation here.
   // TODO: we may allow multiple mutations in a single barrier.
   map<uint32, source.ConnectorSplits> actor_splits = 2;
+  // We may embed a pause mutation here.
+  // TODO: we may allow multiple mutations in a single barrier.
+  bool pause = 4;
 }
 
 message StopMutation {
@@ -171,6 +174,10 @@ message SinkDesc {
   map<string, string> properties = 8;
   catalog.SinkType sink_type = 9;
   repeated plan_common.ColumnCatalog column_catalogs = 10;
+  string db_name = 11;
+  // If the sink is from table or mv, this is name of the table/mv. Otherwise
+  // it is the name of the sink itself.
+  string sink_from_name = 12;
 }
 
 enum SinkLogStoreType {
@@ -449,6 +456,9 @@ message ChainNode {
 
   // The upstream materialized view info used by backfill.
   plan_common.StorageTableDesc table_desc = 7;
+
+  // The rate limit for the chain node.
+  optional uint32 rate_limit = 8;
 }
 
 // BatchPlanNode is used for mv on mv snapshot read.
@@ -530,6 +540,12 @@ message ExpandNode {
 
 message ProjectSetNode {
   repeated expr.ProjectSetSelectItem select_list = 1;
+  // this two field is expressing a list of usize pair, which means when project receives a
+  // watermark with `watermark_input_cols[i]` column index, it should derive a new watermark
+  // with `watermark_output_cols[i]`th expression
+  repeated uint32 watermark_input_cols = 2;
+  repeated uint32 watermark_expr_indices = 3;
+  repeated uint32 nondecreasing_exprs = 4;
 }
 
 // Sorts inputs and outputs ordered data based on watermark.
@@ -581,12 +597,20 @@ message EowcOverWindowNode {
   catalog.Table state_table = 4;
 }
 
-// Note this is not exactly the same as EowcOverWindowNode in terms of future extension, so better to split as 2 different nodes.
+enum OverWindowCachePolicy {
+  OVER_WINDOW_CACHE_POLICY_UNSPECIFIED = 0;
+  OVER_WINDOW_CACHE_POLICY_FULL = 1;
+  OVER_WINDOW_CACHE_POLICY_RECENT = 2;
+  OVER_WINDOW_CACHE_POLICY_RECENT_FIRST_N = 3;
+  OVER_WINDOW_CACHE_POLICY_RECENT_LAST_N = 4;
+}
+
 message OverWindowNode {
   repeated expr.WindowFunction calls = 1;
   repeated uint32 partition_by = 2;
   repeated common.ColumnOrder order_by = 3;
   catalog.Table state_table = 4;
+  OverWindowCachePolicy cache_policy = 5;
 }
 
 message StreamNode {
diff --git a/risedev.yml b/risedev.yml
index 4ba871da0f81a..000971004bf51 100644
--- a/risedev.yml
+++ b/risedev.yml
@@ -20,14 +20,14 @@ profile:
     # config-path: src/config/example.toml
     steps:
       # If you want to use the local s3 storage, enable the following line
-      # - use: minio
+      - use: minio
 
       # If you want to use aws-s3, configure AK and SK in env var and enable the following lines:
       # - use: aws-s3
       #   bucket: test-bucket
 
       # If you want to create CDC source table, uncomment the following line
-      # - use: connector-node
+      - use: connector-node
 
       # if you want to enable etcd backend, uncomment the following lines.
       # - use: etcd
@@ -43,7 +43,7 @@ profile:
       - use: frontend
 
       # If you want to enable compactor, uncomment the following line, and enable either minio or aws-s3 as well.
-      # - use: compactor
+      - use: compactor
 
       # If you want to create source from Kafka, uncomment the following lines
       # Note that kafka depends on zookeeper, so zookeeper must be started beforehand.
@@ -180,11 +180,10 @@ profile:
       - use: meta-node
       - use: compute-node
       - use: frontend
-      # If you want to use hdfs as storage backend, configure hdfs namenode and root path:
+      # If you want to use hdfs as storage backend, configure hdfs namenode:
       - use: opendal
         engine: hdfs
         namenode: "127.0.0.1:9000"
-        root: risingwave
       - use: compactor
       # - use: prometheus
       # - use: grafana
@@ -194,11 +193,10 @@ profile:
       - use: meta-node
       - use: compute-node
       - use: frontend
-      # If you want to use webhdfs as storage backend, configure hdfs namenode and root path:
+      # If you want to use webhdfs as storage backend, configure hdfs namenode:
       - use: opendal
         engine: webhdfs
         namenode: "127.0.0.1:9870"
-        root: risingwave
       - use: compactor
       # - use: prometheus
       # - use: grafana
@@ -209,11 +207,10 @@ profile:
       - use: meta-node
       - use: compute-node
       - use: frontend
-      # If you want to use google cloud storage as storage backend, configure bucket name and root path:
+      # If you want to use google cloud storage as storage backend, configure bucket name:
       - use: opendal
         engine: gcs
         bucket: bucket-name
-        root: risingwave
       - use: compactor
       # - use: prometheus
       # - use: grafana
@@ -224,11 +221,10 @@ profile:
       - use: meta-node
       - use: compute-node
       - use: frontend
-      # If you want to use oss as storage backend, configure bucket name and root path:
+      # If you want to use oss as storage backend, configure bucket name:
       - use: opendal
         engine: oss
-        bucket: test-bucket
-        root: risingwave
+        bucket: bucket-name
       - use: compactor
       # - use: prometheus
       # - use: grafana
@@ -239,11 +235,10 @@ profile:
       - use: meta-node
       - use: compute-node
       - use: frontend
-      # If you want to use azblob as storage backend, configure bucket(container) name and root path:
+      # If you want to use azblob as storage backend, configure bucket(container) name:
       - use: opendal
         engine: azblob
         bucket: test-bucket
-        root: risingwave
       - use: compactor
       # - use: prometheus
       # - use: grafana
@@ -575,7 +570,7 @@ profile:
       - use: compactor
 
   ci-3cn-3fe-in-memory:
-    config-path: src/config/ci.toml
+    config-path: src/config/ci-mem.toml
     steps:
       - use: meta-node
         enable-in-memory-kv-state-backend: true
@@ -610,7 +605,6 @@ profile:
       - use: opendal
         engine: fs
         bucket: ""
-        root: /tmp/rw_ci
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -821,7 +815,7 @@ profile:
       - use: compactor
 
   ci-backfill:
-    config-path: "src/config/backfill.toml"
+    config-path: "src/config/ci-backfill.toml"
     steps:
       - use: minio
       - use: etcd
@@ -1173,8 +1167,6 @@ template:
 
     bucket: risingwave-test
 
-    root: risingwave
-
   # aws-s3 is a placeholder service to provide configurations
   aws-s3:
     # Id to be picked-up by services
diff --git a/scripts/check/check-trailing-spaces.sh b/scripts/check/check-trailing-spaces.sh
index 4876a2265e7d9..b2aba451365c6 100755
--- a/scripts/check/check-trailing-spaces.sh
+++ b/scripts/check/check-trailing-spaces.sh
@@ -3,6 +3,8 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
+self=$0
+
 # Shell colors
 RED='\033[0;31m'
 BLUE='\033[0;34m'
@@ -11,8 +13,12 @@ ORANGE='\033[0;33m'
 BOLD='\033[1m'
 NONE='\033[0m'
 
-_echo_err() {
-    echo -e "${RED}$@${NONE}"
+print_help() {
+    echo "Usage: $self [-f|--fix]"
+    echo
+    echo "Options:"
+    echo "  -f, --fix              Fix trailing spaces."
+    echo "  -h, --help             Show this help message and exit."
 }
 
 fix=false
@@ -21,42 +27,42 @@ while [ $# -gt 0 ]; do
     -f | --fix)
         fix=true
         ;;
+    -h | --help)
+        print_help
+        exit 0
+        ;;
     *)
-        _echo_err "$self: invalid option \`$1\`\n"
+        echo -e "${RED}${BOLD}$self: invalid option \`$1\`\n${NONE}"
+        print_help
         exit 1
         ;;
     esac
     shift
 done
 
-# The following is modified from https://github.com/raisedevs/find-trailing-whitespace/blob/restrict-to-plaintext-only/entrypoint.sh.
+temp_file=$(mktemp)
 
-has_trailing_spaces=false
+echo -ne "${BLUE}"
+git grep -nIP --untracked '[[:space:]]+$' -- ':!src/tests/regress/data' | tee $temp_file || true
+echo -ne "${NONE}"
 
-for file in $(git grep --cached -Il '' -- ':!src/tests/regress/data'); do
-    lines=$(grep -E -rnIH "[[:space:]]+$" "$file" | cut -f-2 -d ":" || echo "")
-    if [ ! -z "$lines" ]; then
-        if [[ $has_trailing_spaces == false ]]; then
-            echo -e "\nLines containing trailing whitespace:\n"
-            has_trailing_spaces=true
-        fi
-        if [[ $fix == true ]]; then
+bad_files=$(cat $temp_file | cut -f1 -d ':' | sort -u)
+rm $temp_file
+
+if [ ! -z "$bad_files" ]; then
+    if [[ $fix == true ]]; then
+        for file in $bad_files; do
             sed -i '' -e's/[[:space:]]*$//' "$file"
-        fi
-        echo -e "${BLUE}$lines${NONE}"
-    fi
-done
+        done
 
-if [[ $has_trailing_spaces == true ]]; then
-    if [[ $fix == false ]]; then
         echo
-        echo -e "${RED}${BOLD}Please clean all the trailing spaces.${NONE}"
-        echo -e "${BOLD}You can run 'scripts/check-trailing-spaces.sh --fix' for convenience.${NONE}"
-        exit 1
+        echo -e "${GREEN}${BOLD}All trailing spaces listed above have been cleaned.${NONE}"
+        exit 0
     else
         echo
-        echo -e "${GREEN}${BOLD}All trailing spaces have been cleaned.${NONE}"
-        exit 0
+        echo -e "${RED}${BOLD}Please clean all the trailing spaces listed above.${NONE}"
+        echo -e "${BOLD}You can run '$self --fix' for convenience.${NONE}"
+        exit 1
     fi
 else
     echo -e "${GREEN}${BOLD}No trailing spaces found.${NONE}"
diff --git a/scripts/source/prepare_ci_kafka.sh b/scripts/source/prepare_ci_kafka.sh
index 0b773933f0f9d..cdc670a572967 100755
--- a/scripts/source/prepare_ci_kafka.sh
+++ b/scripts/source/prepare_ci_kafka.sh
@@ -54,7 +54,7 @@ done
 wait
 
 echo "Fulfill kafka topics"
-python3 -m pip install requests fastavro confluent_kafka
+python3 -m pip install requests fastavro confluent_kafka jsonschema
 for filename in $kafka_data_files; do
     ([ -e "$filename" ]
     base=$(basename "$filename")
@@ -65,7 +65,9 @@ for filename in $kafka_data_files; do
     if [[ "$topic" = *bin ]]; then
         ${KCAT_BIN} -P -b message_queue:29092 -t "$topic" "$filename"
     elif [[ "$topic" = *avro_json ]]; then
-        python3 source/avro_producer.py "message_queue:29092" "http://message_queue:8081" "$filename" "topic"
+        python3 source/schema_registry_producer.py "message_queue:29092" "http://message_queue:8081" "$filename" "topic" "avro"
+    elif [[ "$topic" = *json_schema ]]; then
+        python3 source/schema_registry_producer.py "kafka:9093" "http://schemaregistry:8082" "$filename" "topic" "json"
     else
         cat "$filename" | ${KCAT_BIN} -P -K ^  -b message_queue:29092 -t "$topic"
     fi
@@ -75,9 +77,9 @@ done
 # write schema with name strategy
 
 ## topic: upsert_avro_json-record, key subject: string, value subject: CPLM.OBJ_ATTRIBUTE_VALUE
-(python3 source/avro_producer.py  "message_queue:29092" "http://message_queue:8081" source/test_data/upsert_avro_json.1 "record") &
+(python3 source/schema_registry_producer.py  "message_queue:29092" "http://message_queue:8081" source/test_data/upsert_avro_json.1 "record" "avro") &
 ## topic: upsert_avro_json-topic-record,
 ## key subject: upsert_avro_json-topic-record-string
 ## value subject: upsert_avro_json-topic-record-CPLM.OBJ_ATTRIBUTE_VALUE
-(python3 source/avro_producer.py  "message_queue:29092" "http://message_queue:8081" source/test_data/upsert_avro_json.1 "topic-record") &
+(python3 source/schema_registry_producer.py  "message_queue:29092" "http://message_queue:8081" source/test_data/upsert_avro_json.1 "topic-record" "avro") &
 wait
diff --git a/scripts/source/prepare_ci_pubsub/Cargo.toml b/scripts/source/prepare_ci_pubsub/Cargo.toml
index c7461432888e4..54db02f017287 100644
--- a/scripts/source/prepare_ci_pubsub/Cargo.toml
+++ b/scripts/source/prepare_ci_pubsub/Cargo.toml
@@ -13,7 +13,7 @@ normal = ["workspace-hack"]
 [dependencies]
 anyhow = "1"
 google-cloud-googleapis = { version = "0.11", features = ["pubsub"] }
-google-cloud-pubsub = "0.19"
+google-cloud-pubsub = "0.20"
 tokio = { version = "0.2", package = "madsim-tokio", features = [
     "rt",
     "rt-multi-thread",
diff --git a/scripts/source/avro_producer.py b/scripts/source/schema_registry_producer.py
similarity index 67%
rename from scripts/source/avro_producer.py
rename to scripts/source/schema_registry_producer.py
index 5580d2cd1d51e..f2a58fe1628ed 100644
--- a/scripts/source/avro_producer.py
+++ b/scripts/source/schema_registry_producer.py
@@ -4,6 +4,7 @@
 from confluent_kafka.schema_registry import SchemaRegistryClient, record_subject_name_strategy, \
     topic_record_subject_name_strategy
 from confluent_kafka.schema_registry.avro import AvroSerializer
+from confluent_kafka.schema_registry.json_schema import JSONSerializer
 import sys
 import json
 import os
@@ -24,12 +25,13 @@ def delivery_report(err, msg):
 
 
 if __name__ == '__main__':
-    if len(sys.argv) < 4:
-        print("datagen.py <brokerlist> <schema-registry-url> <file> <name-strategy>")
+    if len(sys.argv) < 5:
+        print("datagen.py <brokerlist> <schema-registry-url> <file> <name-strategy> <json/avro>")
     broker_list = sys.argv[1]
     schema_registry_url = sys.argv[2]
     file = sys.argv[3]
     topic = os.path.basename(file).split(".")[0]
+    type = sys.argv[5]
     if sys.argv[4] not in ['topic', 'record', 'topic-record', '', None]:
         print("name strategy must be one of: topic, record, topic_record")
         exit(1)
@@ -39,7 +41,9 @@ def delivery_report(err, msg):
     print("broker_list: {}".format(broker_list))
     print("schema_registry_url: {}".format(schema_registry_url))
     print("topic: {}".format(topic))
-    print("name strategy: {}".format(sys.argv[4] if sys.argv[4] is not None else 'topic'))
+    print("name strategy: {}".format(
+        sys.argv[4] if sys.argv[4] is not None else 'topic'))
+    print("type: {}".format(type))
 
     schema_registry_conf = {'url': schema_registry_url}
     kafka_conf = {'bootstrap.servers': broker_list}
@@ -60,15 +64,25 @@ def delivery_report(err, msg):
             if i == 0:
                 parts = line.split("^")
                 if len(parts) > 1:
-                    key_serializer = AvroSerializer(schema_registry_client=schema_registry_client,
-                                                    schema_str=parts[0],
-                                                    conf=avro_ser_conf)
-                    value_serializer = AvroSerializer(schema_registry_client=schema_registry_client,
-                                                      schema_str=parts[1],
-                                                      conf=avro_ser_conf)
+                    if type == 'avro':
+                        key_serializer = AvroSerializer(schema_registry_client=schema_registry_client,
+                                                        schema_str=parts[0],
+                                                        conf=avro_ser_conf)
+                        value_serializer = AvroSerializer(schema_registry_client=schema_registry_client,
+                                                          schema_str=parts[1],
+                                                          conf=avro_ser_conf)
+                    else:
+                        key_serializer = JSONSerializer(schema_registry_client=schema_registry_client,
+                                                        schema_str=parts[0])
+                        value_serializer = JSONSerializer(schema_registry_client=schema_registry_client,
+                                                          schema_str=parts[1])
                 else:
-                    value_serializer = AvroSerializer(schema_registry_client=schema_registry_client,
-                                                      schema_str=parts[0])
+                    if type == 'avro':
+                        value_serializer = AvroSerializer(schema_registry_client=schema_registry_client,
+                                                          schema_str=parts[0])
+                    else:
+                        value_serializer = JSONSerializer(schema_registry_client=schema_registry_client,
+                                                          schema_str=parts[0])
             else:
                 parts = line.split("^")
                 if len(parts) > 1:
diff --git a/scripts/source/test_data/json_schema_without_schema.1 b/scripts/source/test_data/json_schema_without_schema.1
new file mode 100644
index 0000000000000..732162bf96ab7
--- /dev/null
+++ b/scripts/source/test_data/json_schema_without_schema.1
@@ -0,0 +1 @@
+{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}}
\ No newline at end of file
diff --git a/scripts/source/test_data/kafka_json_schema.1 b/scripts/source/test_data/kafka_json_schema.1
new file mode 100644
index 0000000000000..e71485885a654
--- /dev/null
+++ b/scripts/source/test_data/kafka_json_schema.1
@@ -0,0 +1,2 @@
+{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productId":{"description":"The unique identifier for a product","type":"integer"},"productName":{"description":"Name of the product","type":"string"},"price":{"description":"The price of the product","type":"number","exclusiveMinimum":0},"tags":{"description":"Tags for the product","type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true},"dimensions":{"type":"object","properties":{"length":{"type":"number"},"width":{"type":"number"},"height":{"type":"number"}},"required":["length","width","height"]}},"required":["productId","productName","price"]}
+{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}}
\ No newline at end of file
diff --git a/scripts/source/test_data/kafka_upsert_json_schema.1 b/scripts/source/test_data/kafka_upsert_json_schema.1
new file mode 100644
index 0000000000000..0f96ceb42a5a0
--- /dev/null
+++ b/scripts/source/test_data/kafka_upsert_json_schema.1
@@ -0,0 +1,4 @@
+{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"id":{"description":"The unique identifier for a product","type":"integer"}},"required":["id"]}^{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"id":{"description":"The unique identifier for a product","type":"integer"},"productName":{"description":"Name of the product","type":"string"},"price":{"description":"The price of the product","type":"number","exclusiveMinimum":0},"tags":{"description":"Tags for the product","type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true},"dimensions":{"type":"object","properties":{"length":{"type":"number"},"width":{"type":"number"},"height":{"type":"number"}},"required":["length","width","height"]}},"required":["id","productName","price"]}
+{"id":1}^{"id":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}}
+{"id":2}^{"id":2,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}}
+{"id":1}^{"id":1,"productName":"An ice sculpture","price":23.0,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}}
\ No newline at end of file
diff --git a/src/README.md b/src/README.md
index 3467d976854ca..b359a1f8f6bbe 100644
--- a/src/README.md
+++ b/src/README.md
@@ -17,3 +17,61 @@ Almost all components of RisingWave are developed in rust, and they are split to
 [state-store]: https://github.com/risingwavelabs/risingwave/blob/main/docs/state-store-overview.md
 [meta-service]: https://github.com/risingwavelabs/risingwave/blob/main/docs/meta-service.md
 [risedev]: https://github.com/risingwavelabs/risingwave/tree/main/src/risedevtool
+
+## Understanding RisingWave Macros
+
+RisingWave uses a lot of macros to simplify development.
+You may choose the read to macro definition directly,
+but something complementary / easier is to use `cargo expand`
+to expand the macro and read the expanded code.
+
+You can also setup `rust-analyzer` in the editor to expand macros on the fly.
+See https://rust-analyzer.github.io/manual.html#expand-macro-recursively.
+This is the **better option** if you want to understand the code interactively.
+
+### Example 1: declarative macro `commit_meta!()`
+
+For instance, within `meta::manager::catalog::CatalogManager::finish_create_table_procedure`
+there's a call to the macro `commit_meta!()`.
+
+To understand it, first I dump the expanded code to a file:
+
+```bash
+cargo expand -p risingwave_meta > meta.rs
+```
+
+Then I search for the function call `finish_create_table_procedure` in my editor,
+and compare the original code with the expanded code in `meta.rs`.
+
+From there I can see that it does the following, followed by some instrumentation code which can be ignored:
+```text
+async {
+    tables.apply_to_txn(&mut trx)?;
+    self.env.meta_store().txn(trx).await?;
+    tables.commit();
+    MetaResult::Ok(())
+}.instrument(/* ... */)
+```
+
+### Example 2: Procedural macro `aggregate`
+
+The other example would be the `#[aggregate]` procedural macro.
+
+To understand it, first I dump the expanded code to a file:
+
+```bash
+cargo expand -p risingwave_expr > expr.rs
+```
+
+Then we identify the `#[aggregate]` macro call to examine. In this case it's `string_agg`.
+
+First, we know that `cargo expand` merges the individual modules into one big file, namespaced by the module name.
+
+In this case `string_agg` is its own module, so we can search for `mod string_agg` in the `expr.rs` we just generated.
+
+We can see that the expanded `string_agg` now also contains `extern fn string_agg_varchar_varchar_varchar`.
+
+Reading the code we understand now that it does the following:
+1. Register it to a global registry for aggregates.
+2. "it" here refers to an `AggFuncSig`, whose definition we can checkout.
+3. Of note is that it will generate an anonymous struct which implements the `AggregateFunction` trait.
\ No newline at end of file
diff --git a/src/batch/src/executor/row_seq_scan.rs b/src/batch/src/executor/row_seq_scan.rs
index b4aa5e4fa4786..3c46682c139de 100644
--- a/src/batch/src/executor/row_seq_scan.rs
+++ b/src/batch/src/executor/row_seq_scan.rs
@@ -34,7 +34,7 @@ use risingwave_pb::common::BatchQueryEpoch;
 use risingwave_pb::plan_common::StorageTableDesc;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::table::batch_table::storage_table::StorageTable;
-use risingwave_storage::table::{collect_data_chunk, get_second, Distribution};
+use risingwave_storage::table::{collect_data_chunk, Distribution};
 use risingwave_storage::{dispatch_state_store, StateStore};
 
 use crate::executor::{
@@ -416,8 +416,7 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
                 ordered,
                 PrefetchOptions::new_for_exhaust_iter(),
             )
-            .await?
-            .map(get_second);
+            .await?;
 
         pin_mut!(iter);
         loop {
diff --git a/src/batch/src/task/task_manager.rs b/src/batch/src/task/task_manager.rs
index ceb0b080db599..858e9bc432b96 100644
--- a/src/batch/src/task/task_manager.rs
+++ b/src/batch/src/task/task_manager.rs
@@ -436,34 +436,6 @@ mod tests {
         assert!(!manager.tasks.lock().contains_key(&task_id));
     }
 
-    #[tokio::test]
-    async fn test_task_cancel_for_block() {
-        let manager = Arc::new(BatchManager::new(
-            BatchConfig::default(),
-            BatchManagerMetrics::for_test(),
-        ));
-        let plan = PlanFragment {
-            root: Some(PlanNode {
-                children: vec![],
-                identity: "".to_string(),
-                node_body: Some(NodeBody::BlockExecutor(true)),
-            }),
-            exchange_info: Some(ExchangeInfo {
-                mode: DistributionMode::Single as i32,
-                distribution: None,
-            }),
-        };
-        let task_id = PbTaskId {
-            query_id: "".to_string(),
-            stage_id: 0,
-            task_id: 0,
-        };
-        manager.fire_task_for_test(&task_id, plan).await.unwrap();
-        manager.cancel_task(&task_id);
-        let task_id = TaskId::from(&task_id);
-        assert!(!manager.tasks.lock().contains_key(&task_id));
-    }
-
     #[tokio::test]
     async fn test_task_abort_for_busy_loop() {
         let manager = Arc::new(BatchManager::new(
@@ -496,37 +468,4 @@ mod tests {
             .abort("Abort Test".to_owned());
         assert!(manager.wait_until_task_aborted(&task_id).await.is_ok());
     }
-
-    #[tokio::test]
-    async fn test_task_abort_for_block() {
-        let manager = Arc::new(BatchManager::new(
-            BatchConfig::default(),
-            BatchManagerMetrics::for_test(),
-        ));
-        let plan = PlanFragment {
-            root: Some(PlanNode {
-                children: vec![],
-                identity: "".to_string(),
-                node_body: Some(NodeBody::BlockExecutor(true)),
-            }),
-            exchange_info: Some(ExchangeInfo {
-                mode: DistributionMode::Single as i32,
-                distribution: None,
-            }),
-        };
-        let task_id = PbTaskId {
-            query_id: "".to_string(),
-            stage_id: 0,
-            task_id: 0,
-        };
-        manager.fire_task_for_test(&task_id, plan).await.unwrap();
-        let task_id = TaskId::from(&task_id);
-        manager
-            .tasks
-            .lock()
-            .get(&task_id)
-            .unwrap()
-            .abort("Abort Test".to_owned());
-        assert!(manager.wait_until_task_aborted(&task_id).await.is_ok());
-    }
 }
diff --git a/src/bench/s3_bench/main.rs b/src/bench/s3_bench/main.rs
index 8b9ecb204268d..f6cbb7cf89200 100644
--- a/src/bench/s3_bench/main.rs
+++ b/src/bench/s3_bench/main.rs
@@ -136,7 +136,7 @@ enum Case {
 
 #[derive(Parser, Debug)]
 pub struct Config {
-    /// AWS S3 Bucket, either <BUCKET> should be given.
+    /// AWS S3 Bucket.
     #[clap(short, long)]
     bucket: String,
 
diff --git a/src/cmd_all/scripts/standalone-demo-dev.sh b/src/cmd_all/scripts/standalone-demo-dev.sh
index 5b4ab18830970..6a4c124ae74e9 100755
--- a/src/cmd_all/scripts/standalone-demo-dev.sh
+++ b/src/cmd_all/scripts/standalone-demo-dev.sh
@@ -22,7 +22,7 @@ cargo run -p risingwave_cmd_all \
                   --listen-addr 127.0.0.1:5688 \
                   --prometheus-listener-addr 127.0.0.1:1222 \
                   --advertise-addr 127.0.0.1:5688 \
-                  --metrics-level 1 \
+                  --metrics-level info \
                   --async-stack-trace verbose \
                   --connector-rpc-endpoint 127.0.0.1:50051 \
                   --parallelism 4 \
@@ -35,5 +35,5 @@ cargo run -p risingwave_cmd_all \
                  --advertise-addr 127.0.0.1:4566 \
                  --prometheus-listener-addr 127.0.0.1:2222 \
                  --health-check-listener-addr 127.0.0.1:6786 \
-                 --metrics-level 1 \
+                 --metrics-level info \
                  --meta-addr http://127.0.0.1:5690"
\ No newline at end of file
diff --git a/src/cmd_all/scripts/standalone-demo-full.sh b/src/cmd_all/scripts/standalone-demo-full.sh
index c207404c1f0b7..83358fd9f3704 100755
--- a/src/cmd_all/scripts/standalone-demo-full.sh
+++ b/src/cmd_all/scripts/standalone-demo-full.sh
@@ -29,7 +29,7 @@ start_standalone() {
                     --listen-addr 127.0.0.1:5688 \
                     --prometheus-listener-addr 127.0.0.1:1222 \
                     --advertise-addr 127.0.0.1:5688 \
-                    --metrics-level 1 \
+                    --metrics-level info \
                     --async-stack-trace verbose \
                     --connector-rpc-endpoint 127.0.0.1:50051 \
                     --parallelism 4 \
@@ -42,7 +42,7 @@ start_standalone() {
                    --advertise-addr 127.0.0.1:4566 \
                    --prometheus-listener-addr 127.0.0.1:2222 \
                    --health-check-listener-addr 127.0.0.1:6786 \
-                   --metrics-level 1 \
+                   --metrics-level info \
                    --meta-addr http://127.0.0.1:5690"
 }
 
diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml
index e3ddbd14cd01c..99a0887d567ae 100644
--- a/src/common/Cargo.toml
+++ b/src/common/Cargo.toml
@@ -16,9 +16,9 @@ normal = ["workspace-hack"]
 [dependencies]
 anyhow = "1"
 arc-swap = "1"
-arrow-array = "45"
-arrow-buffer = "45"
-arrow-schema = "45"
+arrow-array = { workspace = true }
+arrow-buffer = { workspace = true }
+arrow-schema = { workspace = true }
 async-trait = "0.1"
 auto_enums = "0.8"
 bitflags = "2"
@@ -43,6 +43,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] }
 hex = "0.4.3"
 http = "0.2"
 humantime = "2.1"
+hyper = "0.14"
 hytra = { workspace = true }
 itertools = "0.11"
 itoa = "1.0"
@@ -55,6 +56,7 @@ opentelemetry = { version = "0.20", default-features = false }
 parking_lot = "0.12"
 parse-display = "0.8"
 paste = "1"
+pin-project-lite = "0.2"
 postgres-types = { version = "0.2.6", features = [
     "derive",
     "with-chrono-0_4",
@@ -93,14 +95,17 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
 toml = "0.7"
 tonic = { workspace = true }
 tracing = "0.1"
-tracing-opentelemetry = "0.20"
+tracing-opentelemetry = "0.21"
 tracing-subscriber = "0.3.17"
 twox-hash = "1"
 url = "2"
-uuid = "1.4.1"
+uuid = { version = "1", features = ["v4"] }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }
+http-body = "0.4.5"
+tower-layer = "0.3.2"
+tower-service = "0.3.2"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.15", default-features = false }
diff --git a/src/common/proc_macro/src/lib.rs b/src/common/proc_macro/src/lib.rs
index 6d57b99b3b65a..060ee1950624e 100644
--- a/src/common/proc_macro/src/lib.rs
+++ b/src/common/proc_macro/src/lib.rs
@@ -43,6 +43,7 @@ mod estimate_size;
 ///
 /// will generate
 ///
+/// ```ignore
 /// impl OverrideConfig for Opts {
 ///     fn r#override(self, config: &mut RwConfig) {
 ///         if let Some(v) = self.required_str {
diff --git a/src/common/src/array/arrow.rs b/src/common/src/array/arrow.rs
index 9aaec9acbf1fe..9b4165b608d98 100644
--- a/src/common/src/array/arrow.rs
+++ b/src/common/src/array/arrow.rs
@@ -184,7 +184,7 @@ impl TryFrom<&DataType> for arrow_schema::DataType {
             DataType::Timestamp => Ok(Self::Timestamp(arrow_schema::TimeUnit::Microsecond, None)),
             DataType::Timestamptz => Ok(Self::Timestamp(
                 arrow_schema::TimeUnit::Microsecond,
-                Some("UTC".into()),
+                Some("+00:00".into()),
             )),
             DataType::Time => Ok(Self::Time64(arrow_schema::TimeUnit::Microsecond)),
             DataType::Interval => Ok(Self::Interval(arrow_schema::IntervalUnit::MonthDayNano)),
diff --git a/src/common/src/array/data_chunk.rs b/src/common/src/array/data_chunk.rs
index 05405d59432a2..e5ae85edbfd99 100644
--- a/src/common/src/array/data_chunk.rs
+++ b/src/common/src/array/data_chunk.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Display;
 use std::hash::BuildHasher;
 use std::sync::Arc;
 use std::{fmt, usize};
 
 use bytes::Bytes;
+use either::Either;
 use itertools::Itertools;
 use rand::rngs::SmallRng;
 use rand::{Rng, SeedableRng};
@@ -65,6 +67,8 @@ pub struct DataChunk {
 }
 
 impl DataChunk {
+    pub(crate) const PRETTY_TABLE_PRESET: &str = "||--+-++|    ++++++";
+
     /// Create a `DataChunk` with `columns` and visibility. The visibility can either be a `Bitmap`
     /// or a simple cardinality number.
     pub fn new<V: Into<Vis>>(columns: Vec<ArrayRef>, vis: V) -> Self {
@@ -392,24 +396,31 @@ impl DataChunk {
         RowRef::new(self, pos)
     }
 
-    /// `to_pretty_string` returns a table-like text representation of the `DataChunk`.
-    pub fn to_pretty_string(&self) -> String {
+    /// Returns a table-like text representation of the `DataChunk`.
+    pub fn to_pretty(&self) -> impl Display {
         use comfy_table::Table;
+
+        if self.cardinality() == 0 {
+            return Either::Left("(empty)");
+        }
+
         let mut table = Table::new();
-        table.load_preset("||--+-++|    ++++++\n");
+        table.load_preset(Self::PRETTY_TABLE_PRESET);
+
         for row in self.rows() {
             let cells: Vec<_> = row
                 .iter()
                 .map(|v| {
                     match v {
-                        None => "".to_owned(), // null
+                        None => "".to_owned(), // NULL
                         Some(scalar) => scalar.to_text(),
                     }
                 })
                 .collect();
             table.add_row(cells);
         }
-        table.to_string()
+
+        Either::Right(table)
     }
 
     /// Keep the specified columns and set the rest elements to null.
@@ -624,7 +635,7 @@ impl fmt::Debug for DataChunk {
             "DataChunk {{ cardinality = {}, capacity = {}, data = \n{} }}",
             self.cardinality(),
             self.capacity(),
-            self.to_pretty_string()
+            self.to_pretty()
         )
     }
 }
@@ -1005,7 +1016,7 @@ mod tests {
             4,
         );
         assert_eq!(
-            chunk.to_pretty_string(),
+            chunk.to_pretty().to_string(),
             "\
 +---+---+
 | 1 | 6 |
diff --git a/src/common/src/array/list_array.rs b/src/common/src/array/list_array.rs
index c4cfe7a36e4be..2c4a8cf042548 100644
--- a/src/common/src/array/list_array.rs
+++ b/src/common/src/array/list_array.rs
@@ -15,6 +15,7 @@
 use core::fmt;
 use std::cmp::Ordering;
 use std::fmt::Debug;
+use std::future::Future;
 use std::hash::Hash;
 use std::mem::size_of;
 
@@ -170,9 +171,9 @@ impl ListArrayBuilder {
 /// Each item of this `ListArray` is a `List<T>`, or called `T[]` (T array).
 ///
 /// * As other arrays, there is a null bitmap, with `1` meaning nonnull and `0` meaning null.
-/// * As [`BytesArray`], there is an offsets `Vec` and a value `Array`. The value `Array` has all
-///   items concatenated, and the offsets `Vec` stores start and end indices into it for slicing.
-///   Effectively, the inner array is the flattened form, and `offsets.len() == n + 1`.
+/// * As [`super::BytesArray`], there is an offsets `Vec` and a value `Array`. The value `Array` has
+///   all items concatenated, and the offsets `Vec` stores start and end indices into it for
+///   slicing. Effectively, the inner array is the flattened form, and `offsets.len() == n + 1`.
 ///
 /// For example, `values (array[1]), (array[]::int[]), (null), (array[2, 3]);` stores an inner
 ///  `I32Array` with `[1, 2, 3]`, along with offsets `[0, 1, 1, 1, 3]` and null bitmap `TTFT`.
@@ -269,6 +270,31 @@ impl ListArray {
         Ok(arr.into())
     }
 
+    /// Apply the function on the underlying elements.
+    /// e.g. `map_inner([[1,2,3],NULL,[4,5]], DOUBLE) = [[2,4,6],NULL,[8,10]]`
+    pub async fn map_inner<E, Fut, F>(self, f: F) -> std::result::Result<ListArray, E>
+    where
+        F: FnOnce(ArrayImpl) -> Fut,
+        Fut: Future<Output = std::result::Result<ArrayImpl, E>>,
+    {
+        let Self {
+            bitmap,
+            offsets,
+            value,
+            ..
+        } = self;
+
+        let new_value = (f)(*value).await?;
+        let new_value_type = new_value.data_type();
+
+        Ok(Self {
+            offsets,
+            bitmap,
+            value: Box::new(new_value),
+            value_type: new_value_type,
+        })
+    }
+
     // Used for testing purposes
     pub fn from_iter(
         values: impl IntoIterator<Item = Option<ArrayImpl>>,
diff --git a/src/common/src/array/stream_chunk.rs b/src/common/src/array/stream_chunk.rs
index f214fabfa5fd5..e280af1257c23 100644
--- a/src/common/src/array/stream_chunk.rs
+++ b/src/common/src/array/stream_chunk.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Display;
 use std::mem::size_of;
 use std::ops::{Deref, DerefMut};
 use std::sync::Arc;
 use std::{fmt, mem};
 
+use either::Either;
 use itertools::Itertools;
 use rand::prelude::SmallRng;
 use rand::{Rng, SeedableRng};
@@ -26,6 +28,7 @@ use super::vis::VisMut;
 use super::{ArrayImpl, ArrayRef, ArrayResult, DataChunkTestExt};
 use crate::array::{DataChunk, Vis};
 use crate::buffer::Bitmap;
+use crate::catalog::Schema;
 use crate::estimate_size::EstimateSize;
 use crate::field_generator::VarcharProperty;
 use crate::row::Row;
@@ -199,16 +202,34 @@ impl StreamChunk {
         &self.ops
     }
 
-    /// `to_pretty_string` returns a table-like text representation of the `StreamChunk`.
-    pub fn to_pretty_string(&self) -> String {
+    /// Returns a table-like text representation of the `StreamChunk`.
+    pub fn to_pretty(&self) -> impl Display {
+        self.to_pretty_inner(None)
+    }
+
+    /// Returns a table-like text representation of the `StreamChunk` with a header of column names
+    /// from the given `schema`.
+    pub fn to_pretty_with_schema(&self, schema: &Schema) -> impl Display {
+        self.to_pretty_inner(Some(schema))
+    }
+
+    fn to_pretty_inner(&self, schema: Option<&Schema>) -> impl Display {
         use comfy_table::{Cell, CellAlignment, Table};
 
         if self.cardinality() == 0 {
-            return "(empty)".to_owned();
+            return Either::Left("(empty)");
         }
 
         let mut table = Table::new();
-        table.load_preset("||--+-++|    ++++++");
+        table.load_preset(DataChunk::PRETTY_TABLE_PRESET);
+
+        if let Some(schema) = schema {
+            assert_eq!(self.dimension(), schema.len());
+            let cells = std::iter::once(String::new())
+                .chain(schema.fields().iter().map(|f| f.name.clone()));
+            table.set_header(cells);
+        }
+
         for (op, row_ref) in self.rows() {
             let mut cells = Vec::with_capacity(row_ref.len() + 1);
             cells.push(
@@ -229,7 +250,8 @@ impl StreamChunk {
             }
             table.add_row(cells);
         }
-        table.to_string()
+
+        Either::Right(table)
     }
 
     /// Reorder (and possibly remove) columns.
@@ -290,7 +312,7 @@ impl fmt::Debug for StreamChunk {
                 "StreamChunk {{ cardinality: {}, capacity: {}, data: \n{}\n }}",
                 self.cardinality(),
                 self.capacity(),
-                self.to_pretty_string()
+                self.to_pretty()
             )
         } else {
             f.debug_struct("StreamChunk")
@@ -636,7 +658,7 @@ mod tests {
             U+ 4 .",
         );
         assert_eq!(
-            chunk.to_pretty_string(),
+            chunk.to_pretty().to_string(),
             "\
 +----+---+---+
 |  + | 1 | 6 |
diff --git a/src/common/src/cast/mod.rs b/src/common/src/cast/mod.rs
index fdf3c7e598da2..82c69984ec0ea 100644
--- a/src/common/src/cast/mod.rs
+++ b/src/common/src/cast/mod.rs
@@ -43,7 +43,7 @@ pub fn str_to_timestamp(elem: &str) -> Result<Timestamp> {
 
 #[inline]
 pub fn parse_naive_date(s: &str) -> Result<NaiveDate> {
-    let res = SpeedDate::parse_str(s).map_err(|_| PARSE_ERROR_STR_TO_DATE.to_string())?;
+    let res = SpeedDate::parse_str_rfc3339(s).map_err(|_| PARSE_ERROR_STR_TO_DATE.to_string())?;
     Ok(Date::from_ymd_uncheck(res.year as i32, res.month as u32, res.day as u32).0)
 }
 
@@ -63,7 +63,10 @@ pub fn parse_naive_time(s: &str) -> Result<NaiveTime> {
 
 #[inline]
 pub fn parse_naive_datetime(s: &str) -> Result<NaiveDateTime> {
-    if let Ok(res) = SpeedDateTime::parse_str(s) {
+    if let Ok(res) = SpeedDateTime::parse_str_rfc3339(s) {
+        if res.time.tz_offset.is_some() {
+            return Err(PARSE_ERROR_STR_TO_TIMESTAMP.into());
+        }
         Ok(Date::from_ymd_uncheck(
             res.date.year as i32,
             res.date.month as u32,
@@ -77,7 +80,8 @@ pub fn parse_naive_datetime(s: &str) -> Result<NaiveDateTime> {
         )
         .0)
     } else {
-        let res = SpeedDate::parse_str(s).map_err(|_| PARSE_ERROR_STR_TO_TIMESTAMP.to_string())?;
+        let res = SpeedDate::parse_str_rfc3339(s)
+            .map_err(|_| PARSE_ERROR_STR_TO_TIMESTAMP.to_string())?;
         Ok(
             Date::from_ymd_uncheck(res.year as i32, res.month as u32, res.day as u32)
                 .and_hms_micro_uncheck(0, 0, 0, 0)
@@ -238,7 +242,7 @@ mod tests {
         str_to_timestamp("1999-01-08 04:02").unwrap();
         str_to_timestamp("1999-01-08 04:05:06").unwrap();
         assert_eq!(
-            str_to_timestamp("2022-08-03T10:34:02Z").unwrap(),
+            str_to_timestamp("2022-08-03T10:34:02").unwrap(),
             str_to_timestamp("2022-08-03 10:34:02").unwrap()
         );
         str_to_date("1999-01-08").unwrap();
diff --git a/src/common/src/catalog/column.rs b/src/common/src/catalog/column.rs
index e4b327ec101ad..173253f34dc8a 100644
--- a/src/common/src/catalog/column.rs
+++ b/src/common/src/catalog/column.rs
@@ -19,7 +19,7 @@ use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
 use risingwave_pb::plan_common::{PbColumnCatalog, PbColumnDesc};
 
 use super::row_id_column_desc;
-use crate::catalog::{Field, ROW_ID_COLUMN_ID};
+use crate::catalog::{offset_column_desc, Field, ROW_ID_COLUMN_ID};
 use crate::error::ErrorCode;
 use crate::types::DataType;
 
@@ -317,6 +317,13 @@ impl ColumnCatalog {
             is_hidden: true,
         }
     }
+
+    pub fn offset_column() -> Self {
+        Self {
+            column_desc: offset_column_desc(),
+            is_hidden: true,
+        }
+    }
 }
 
 impl From<PbColumnCatalog> for ColumnCatalog {
diff --git a/src/common/src/catalog/mod.rs b/src/common/src/catalog/mod.rs
index 070ad5d9f777e..e83b4aa638907 100644
--- a/src/common/src/catalog/mod.rs
+++ b/src/common/src/catalog/mod.rs
@@ -113,6 +113,27 @@ pub fn row_id_column_desc() -> ColumnDesc {
     }
 }
 
+pub const OFFSET_COLUMN_NAME: &str = "_rw_offset";
+
+pub fn offset_column_name() -> String {
+    OFFSET_COLUMN_NAME.to_string()
+}
+
+pub fn is_offset_column_name(name: &str) -> bool {
+    name.starts_with(OFFSET_COLUMN_NAME)
+}
+/// Creates a offset column for storing upstream offset
+pub fn offset_column_desc() -> ColumnDesc {
+    ColumnDesc {
+        data_type: DataType::Varchar,
+        column_id: ColumnId::placeholder(),
+        name: offset_column_name(),
+        field_descs: vec![],
+        type_name: "".to_string(),
+        generated_or_default_column: None,
+    }
+}
+
 /// The local system catalog reader in the frontend node.
 #[async_trait]
 pub trait SysCatalogReader: Sync + Send + 'static {
diff --git a/src/common/src/config.rs b/src/common/src/config.rs
index eba690c624030..042a7f1708d4c 100644
--- a/src/common/src/config.rs
+++ b/src/common/src/config.rs
@@ -17,6 +17,7 @@
 //! [`RwConfig`] corresponds to the whole config file and each other config struct corresponds to a
 //! section in `risingwave.toml`.
 
+use std::cmp::Ordering;
 use std::collections::BTreeMap;
 use std::fs;
 use std::num::NonZeroUsize;
@@ -358,11 +359,9 @@ pub struct ServerConfig {
     #[serde(default = "default::server::connection_pool_size")]
     pub connection_pool_size: u16,
 
-    #[serde(default = "default::server::metrics_level")]
     /// Used for control the metrics level, similar to log level.
-    /// 0 = close metrics
-    /// >0 = open metrics
-    pub metrics_level: u32,
+    #[serde(default = "default::server::metrics_level")]
+    pub metrics_level: MetricLevel,
 
     #[serde(default = "default::server::telemetry_enabled")]
     pub telemetry_enabled: bool,
@@ -423,6 +422,27 @@ pub struct StreamingConfig {
     pub unrecognized: Unrecognized<Self>,
 }
 
+#[derive(Debug, Default, Clone, Copy, ValueEnum, Serialize, Deserialize)]
+pub enum MetricLevel {
+    #[default]
+    Disabled = 0,
+    Critical = 1,
+    Info = 2,
+    Debug = 3,
+}
+
+impl PartialEq<Self> for MetricLevel {
+    fn eq(&self, other: &Self) -> bool {
+        (*self as u8).eq(&(*other as u8))
+    }
+}
+
+impl PartialOrd for MetricLevel {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        (*self as u8).partial_cmp(&(*other as u8))
+    }
+}
+
 /// The section `[storage]` in `risingwave.toml`.
 #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
 pub struct StorageConfig {
@@ -496,6 +516,9 @@ pub struct StorageConfig {
     #[serde(default)]
     pub meta_file_cache: FileCacheConfig,
 
+    #[serde(default)]
+    pub cache_refill: CacheRefillConfig,
+
     /// Whether to enable streaming upload for sstable.
     #[serde(default = "default::storage::min_sst_size_for_streaming_upload")]
     pub min_sst_size_for_streaming_upload: u64,
@@ -538,17 +561,30 @@ pub struct StorageConfig {
     pub compactor_max_sst_key_count: u64,
     #[serde(default = "default::storage::compact_iter_recreate_timeout_ms")]
     pub compact_iter_recreate_timeout_ms: u64,
-
     #[serde(default = "default::storage::compactor_max_sst_size")]
     pub compactor_max_sst_size: u64,
+    #[serde(default, flatten)]
+    pub unrecognized: Unrecognized<Self>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
+pub struct CacheRefillConfig {
+    #[serde(default = "default::cache_refill::data_refill_levels")]
+    pub data_refill_levels: Vec<u32>,
+
+    #[serde(default = "default::cache_refill::timeout_ms")]
+    pub timeout_ms: u64,
+
+    #[serde(default = "default::cache_refill::concurrency")]
+    pub concurrency: usize,
 
     #[serde(default, flatten)]
     pub unrecognized: Unrecognized<Self>,
 }
 
-/// The subsection `[storage.file_cache]` in `risingwave.toml`.
+/// The subsection `[storage.data_file_cache]` and `[storage.meta_file_cache]` in `risingwave.toml`.
 ///
-/// It's put at [`StorageConfig::file_cache`].
+/// It's put at [`StorageConfig::data_file_cache`] and  [`StorageConfig::meta_file_cache`].
 #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
 pub struct FileCacheConfig {
     #[serde(default = "default::file_cache::dir")]
@@ -593,9 +629,6 @@ pub struct FileCacheConfig {
     #[serde(default = "default::file_cache::reclaim_rate_limit_mb")]
     pub reclaim_rate_limit_mb: usize,
 
-    #[serde(default = "default::file_cache::refill_levels")]
-    pub refill_levels: Vec<u32>,
-
     #[serde(default, flatten)]
     pub unrecognized: Unrecognized<Self>,
 }
@@ -751,6 +784,10 @@ pub struct SystemConfig {
     /// Max number of concurrent creating streaming jobs.
     #[serde(default = "default::system::max_concurrent_creating_streaming_jobs")]
     pub max_concurrent_creating_streaming_jobs: Option<u32>,
+
+    /// Whether to pause all data sources on next bootstrap.
+    #[serde(default = "default::system::pause_on_next_bootstrap")]
+    pub pause_on_next_bootstrap: Option<bool>,
 }
 
 impl SystemConfig {
@@ -768,6 +805,7 @@ impl SystemConfig {
             backup_storage_directory: self.backup_storage_directory,
             telemetry_enabled: self.telemetry_enabled,
             max_concurrent_creating_streaming_jobs: self.max_concurrent_creating_streaming_jobs,
+            pause_on_next_bootstrap: self.pause_on_next_bootstrap,
         }
     }
 }
@@ -870,7 +908,7 @@ pub mod default {
     }
 
     pub mod server {
-        use crate::config::AutoDumpHeapProfileConfig;
+        use crate::config::{AutoDumpHeapProfileConfig, MetricLevel};
 
         pub fn heartbeat_interval_ms() -> u32 {
             1000
@@ -880,8 +918,8 @@ pub mod default {
             16
         }
 
-        pub fn metrics_level() -> u32 {
-            0
+        pub fn metrics_level() -> MetricLevel {
+            MetricLevel::Info
         }
 
         pub fn telemetry_enabled() -> bool {
@@ -894,7 +932,6 @@ pub mod default {
     }
 
     pub mod storage {
-
         pub fn share_buffers_sync_parallelism() -> u32 {
             1
         }
@@ -969,7 +1006,7 @@ pub mod default {
         }
 
         pub fn max_preload_wait_time_mill() -> u64 {
-            10
+            0
         }
 
         pub fn object_store_streaming_read_timeout_ms() -> u64 {
@@ -1076,10 +1113,20 @@ pub mod default {
         pub fn reclaim_rate_limit_mb() -> usize {
             0
         }
+    }
 
-        pub fn refill_levels() -> Vec<u32> {
+    pub mod cache_refill {
+        pub fn data_refill_levels() -> Vec<u32> {
             vec![]
         }
+
+        pub fn timeout_ms() -> u64 {
+            6000
+        }
+
+        pub fn concurrency() -> usize {
+            100
+        }
     }
 
     pub mod auto_dump_heap_profile {
@@ -1136,60 +1183,12 @@ pub mod default {
     }
 
     pub mod system {
-        use crate::system_param;
-
-        pub fn barrier_interval_ms() -> Option<u32> {
-            system_param::default::barrier_interval_ms()
-        }
-
-        pub fn checkpoint_frequency() -> Option<u64> {
-            system_param::default::checkpoint_frequency()
-        }
-
-        pub fn parallel_compact_size_mb() -> Option<u32> {
-            system_param::default::parallel_compact_size_mb()
-        }
-
-        pub fn sstable_size_mb() -> Option<u32> {
-            system_param::default::sstable_size_mb()
-        }
-
-        pub fn block_size_kb() -> Option<u32> {
-            system_param::default::block_size_kb()
-        }
-
-        pub fn bloom_false_positive() -> Option<f64> {
-            system_param::default::bloom_false_positive()
-        }
-
-        pub fn state_store() -> Option<String> {
-            system_param::default::state_store()
-        }
-
-        pub fn data_directory() -> Option<String> {
-            system_param::default::data_directory()
-        }
-
-        pub fn backup_storage_url() -> Option<String> {
-            system_param::default::backup_storage_url()
-        }
-
-        pub fn backup_storage_directory() -> Option<String> {
-            system_param::default::backup_storage_directory()
-        }
-
-        pub fn telemetry_enabled() -> Option<bool> {
-            system_param::default::telemetry_enabled()
-        }
-
-        pub fn max_concurrent_creating_streaming_jobs() -> Option<u32> {
-            system_param::default::max_concurrent_creating_streaming_jobs()
-        }
+        pub use crate::system_param::default::*;
     }
 
     pub mod batch {
         pub fn enable_barrier_read() -> bool {
-            true
+            false
         }
     }
 
diff --git a/src/common/src/hash/key.rs b/src/common/src/hash/key.rs
index 862d776976fae..b4bee5aa83d0b 100644
--- a/src/common/src/hash/key.rs
+++ b/src/common/src/hash/key.rs
@@ -254,6 +254,8 @@ pub type XxHash64HashCode = HashCode<XxHash64Builder>;
 /// hasher.
 ///
 /// WARN: This should ONLY be used along with [`HashKey`].
+///
+/// [`HashKey`]: crate::hash::HashKey
 #[derive(Default)]
 pub struct PrecomputedHasher {
     hash_code: u64,
diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs
index 6de6d63ce18e8..b2d1b8da82063 100644
--- a/src/common/src/lib.rs
+++ b/src/common/src/lib.rs
@@ -40,6 +40,7 @@
 #![feature(impl_trait_in_assoc_type)]
 #![feature(result_option_inspect)]
 #![feature(negative_impls)]
+#![feature(async_fn_in_trait)]
 
 #[macro_use]
 pub mod jemalloc;
diff --git a/src/common/src/memory/mem_context.rs b/src/common/src/memory/mem_context.rs
index d9b5eaa29cace..70aa20d9e331d 100644
--- a/src/common/src/memory/mem_context.rs
+++ b/src/common/src/memory/mem_context.rs
@@ -27,7 +27,7 @@ struct MemoryContextInner {
 #[derive(Clone)]
 pub struct MemoryContext {
     /// Add None op mem context, so that we don't need to return [`Option`] in
-    /// [`BatchTaskContext`]. This helps with later `Allocator` implementation.
+    /// `BatchTaskContext`. This helps with later `Allocator` implementation.
     inner: Option<Arc<MemoryContextInner>>,
 }
 
diff --git a/src/common/src/metrics.rs b/src/common/src/metrics.rs
index effa553b81b5c..357cfa38ff042 100644
--- a/src/common/src/metrics.rs
+++ b/src/common/src/metrics.rs
@@ -25,6 +25,9 @@ use tracing_subscriber::Layer;
 
 use crate::monitor::GLOBAL_METRICS_REGISTRY;
 
+mod relabeled_metric;
+pub use relabeled_metric::*;
+
 #[derive(Debug)]
 pub struct TrAdderAtomic(TrAdder<i64>);
 
diff --git a/src/common/src/metrics/relabeled_metric.rs b/src/common/src/metrics/relabeled_metric.rs
new file mode 100644
index 0000000000000..e620ee30ba844
--- /dev/null
+++ b/src/common/src/metrics/relabeled_metric.rs
@@ -0,0 +1,132 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use prometheus::core::{AtomicU64, GenericCounter, GenericCounterVec};
+use prometheus::{Histogram, HistogramVec};
+
+use crate::config::MetricLevel;
+
+/// For all `Relabeled*Vec` below,
+/// - when `metric_level` <= `relabel_threshold`, they behaves exactly the same as their inner
+///   metric.
+/// - when `metric_level` > `relabel_threshold`, all their input label values are rewrite to "" when
+/// calling `with_label_values`. That's means the metric vec is aggregated into a single metric.
+
+/// These wrapper classes add a `metric_level` field to corresponding metric.
+/// We could have use one single struct to represent all `MetricVec<T: MetricVecBuilder>`, rather
+/// than specializing them one by one. However, that's undoable because prometheus crate doesn't
+/// export `MetricVecBuilder` implementation like `HistogramVecBuilder`.
+#[derive(Clone, Debug)]
+pub struct RelabeledHistogramVec {
+    relabel_threshold: MetricLevel,
+    metric_level: MetricLevel,
+    metric: HistogramVec,
+
+    /// The first `relabel_num` labels will be relabeled to empty string
+    ///
+    /// For example, if `relabel_num` is 1, and the input labels are `["actor_id",
+    /// "fragment_id", "table_id"]`, when threshold is reached, the label values will be
+    /// `["", "<original_fragment_id>", "<original_table_id>"]`.
+    relabel_num: usize,
+}
+
+impl RelabeledHistogramVec {
+    pub fn with_metric_level(
+        metric_level: MetricLevel,
+        metric: HistogramVec,
+        relabel_threshold: MetricLevel,
+    ) -> Self {
+        Self {
+            relabel_threshold,
+            metric_level,
+            metric,
+            relabel_num: usize::MAX,
+        }
+    }
+
+    pub fn with_metric_level_relabel_n(
+        metric_level: MetricLevel,
+        metric: HistogramVec,
+        relabel_threshold: MetricLevel,
+        relabel_num: usize,
+    ) -> Self {
+        Self {
+            relabel_threshold,
+            metric_level,
+            metric,
+            relabel_num,
+        }
+    }
+
+    pub fn with_label_values(&self, vals: &[&str]) -> Histogram {
+        if self.metric_level > self.relabel_threshold {
+            // relabel first n labels to empty string
+            let mut relabeled_vals = vals.to_vec();
+            for label in relabeled_vals.iter_mut().take(self.relabel_num) {
+                *label = "";
+            }
+            return self.metric.with_label_values(&relabeled_vals);
+        }
+        self.metric.with_label_values(vals)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct RelabeledCounterVec {
+    relabel_threshold: MetricLevel,
+    metric_level: MetricLevel,
+    metric: GenericCounterVec<AtomicU64>,
+    relabel_num: usize,
+}
+
+impl RelabeledCounterVec {
+    pub fn with_metric_level(
+        metric_level: MetricLevel,
+        metric: GenericCounterVec<AtomicU64>,
+        relabel_threshold: MetricLevel,
+    ) -> Self {
+        Self {
+            relabel_threshold,
+            metric_level,
+            metric,
+            relabel_num: usize::MAX,
+        }
+    }
+
+    pub fn with_metric_level_relabel_n(
+        metric_level: MetricLevel,
+        metric: GenericCounterVec<AtomicU64>,
+        relabel_threshold: MetricLevel,
+        relabel_num: usize,
+    ) -> Self {
+        Self {
+            relabel_threshold,
+            metric_level,
+            metric,
+            relabel_num,
+        }
+    }
+
+    pub fn with_label_values(&self, vals: &[&str]) -> GenericCounter<AtomicU64> {
+        if self.metric_level > self.relabel_threshold {
+            // relabel first n labels to empty string
+            let mut relabeled_vals = vals.to_vec();
+            for label in relabeled_vals.iter_mut().take(self.relabel_num) {
+                *label = "";
+            }
+            return self.metric.with_label_values(&relabeled_vals);
+        }
+        self.metric.with_label_values(vals)
+    }
+}
diff --git a/src/common/src/monitor/connection.rs b/src/common/src/monitor/connection.rs
new file mode 100644
index 0000000000000..dbe75087dfa21
--- /dev/null
+++ b/src/common/src/monitor/connection.rs
@@ -0,0 +1,684 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::type_name;
+use std::cmp::Ordering;
+use std::future::Future;
+use std::io::{Error, IoSlice};
+use std::pin::Pin;
+use std::sync::LazyLock;
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use futures::FutureExt;
+use http::Uri;
+use hyper::client::connect::Connection;
+use hyper::client::HttpConnector;
+use hyper::service::Service;
+use pin_project_lite::pin_project;
+use prometheus::core::{
+    AtomicI64, AtomicU64, GenericCounter, GenericCounterVec, GenericGauge, GenericGaugeVec,
+};
+use prometheus::{
+    register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, Registry,
+};
+use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
+use tonic::transport::{Channel, Endpoint};
+use tracing::{info, warn};
+
+use crate::monitor::GLOBAL_METRICS_REGISTRY;
+
+pub trait MonitorAsyncReadWrite {
+    fn on_read(&mut self, _size: usize) {}
+    fn on_eof(&mut self) {}
+    fn on_read_err(&mut self, _err: &std::io::Error) {}
+
+    fn on_write(&mut self, _size: usize) {}
+    fn on_flush(&mut self) {}
+    fn on_shutdown(&mut self) {}
+    fn on_write_err(&mut self, _err: &std::io::Error) {}
+}
+
+pin_project! {
+    #[derive(Clone)]
+    pub struct MonitoredConnection<C, M> {
+        #[pin]
+        inner: C,
+        monitor: M,
+    }
+}
+
+impl<C, M> MonitoredConnection<C, M> {
+    pub fn new(connector: C, monitor: M) -> Self {
+        Self {
+            inner: connector,
+            monitor,
+        }
+    }
+
+    fn project_into(this: Pin<&mut Self>) -> (Pin<&mut C>, &mut M) {
+        let this = this.project();
+        (this.inner, this.monitor)
+    }
+}
+
+impl<C: AsyncRead, M: MonitorAsyncReadWrite> AsyncRead for MonitoredConnection<C, M> {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        let before_buf_size = buf.filled().len();
+        let (inner, monitor) = MonitoredConnection::project_into(self);
+        let ret = inner.poll_read(cx, buf);
+        match &ret {
+            Poll::Ready(Ok(())) => {
+                let after_buf_size = buf.filled().len();
+                match after_buf_size.cmp(&before_buf_size) {
+                    Ordering::Less => {
+                        unreachable!(
+                            "buf size decrease after poll read. Bad AsyncRead implementation on {}",
+                            type_name::<C>()
+                        );
+                    }
+                    Ordering::Equal => {
+                        monitor.on_eof();
+                    }
+                    Ordering::Greater => {
+                        monitor.on_read(after_buf_size - before_buf_size);
+                    }
+                }
+            }
+            Poll::Ready(Err(e)) => {
+                monitor.on_read_err(e);
+            }
+            Poll::Pending => {}
+        }
+        ret
+    }
+}
+
+impl<C: AsyncWrite, M: MonitorAsyncReadWrite> AsyncWrite for MonitoredConnection<C, M> {
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<Result<usize, Error>> {
+        let (inner, monitor) = MonitoredConnection::project_into(self);
+        let ret = inner.poll_write(cx, buf);
+        match &ret {
+            Poll::Ready(Ok(size)) => {
+                monitor.on_write(*size);
+            }
+            Poll::Ready(Err(e)) => {
+                monitor.on_write_err(e);
+            }
+            Poll::Pending => {}
+        }
+        ret
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        let (inner, monitor) = MonitoredConnection::project_into(self);
+        let ret = inner.poll_flush(cx);
+        match &ret {
+            Poll::Ready(Ok(())) => {
+                monitor.on_flush();
+            }
+            Poll::Ready(Err(e)) => {
+                monitor.on_write_err(e);
+            }
+            Poll::Pending => {}
+        }
+        ret
+    }
+
+    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        let (inner, monitor) = MonitoredConnection::project_into(self);
+        let ret = inner.poll_shutdown(cx);
+        match &ret {
+            Poll::Ready(result) => {
+                monitor.on_shutdown();
+                if let Err(e) = result {
+                    monitor.on_write_err(e);
+                }
+            }
+            Poll::Pending => {}
+        }
+        ret
+    }
+
+    fn poll_write_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &[IoSlice<'_>],
+    ) -> Poll<Result<usize, Error>> {
+        let (inner, monitor) = MonitoredConnection::project_into(self);
+        let ret = inner.poll_write_vectored(cx, bufs);
+        match &ret {
+            Poll::Ready(Ok(size)) => {
+                monitor.on_write(*size);
+            }
+            Poll::Ready(Err(e)) => {
+                monitor.on_write_err(e);
+            }
+            Poll::Pending => {}
+        }
+        ret
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        self.inner.is_write_vectored()
+    }
+}
+
+impl<C: Connection, M> Connection for MonitoredConnection<C, M> {
+    fn connected(&self) -> hyper::client::connect::Connected {
+        self.inner.connected()
+    }
+}
+
+#[cfg(not(madsim))]
+impl<C: tonic::transport::server::Connected, M> tonic::transport::server::Connected
+    for MonitoredConnection<C, M>
+{
+    type ConnectInfo = C::ConnectInfo;
+
+    fn connect_info(&self) -> Self::ConnectInfo {
+        self.inner.connect_info()
+    }
+}
+
+pub trait MonitorNewConnection {
+    type ConnectionMonitor: MonitorAsyncReadWrite;
+
+    fn new_connection_monitor(&self, endpoint: String) -> Self::ConnectionMonitor;
+    fn on_err(&self, endpoint: String);
+}
+
+impl<C: Service<Uri>, M: MonitorNewConnection + Clone + 'static> Service<Uri>
+    for MonitoredConnection<C, M>
+where
+    C::Future: 'static,
+{
+    type Error = C::Error;
+    type Response = MonitoredConnection<C::Response, M::ConnectionMonitor>;
+
+    type Future = impl Future<Output = Result<Self::Response, Self::Error>> + 'static;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        let ret = self.inner.poll_ready(cx);
+        if let Poll::Ready(Err(_)) = &ret {
+            self.monitor.on_err("<poll_ready>".to_string());
+        }
+        ret
+    }
+
+    fn call(&mut self, uri: Uri) -> Self::Future {
+        let endpoint = format!("{:?}:{:?}", uri.host(), uri.port());
+        let monitor = self.monitor.clone();
+        self.inner
+            .call(uri)
+            .map(move |result: Result<_, _>| match result {
+                Ok(resp) => Ok(MonitoredConnection::new(
+                    resp,
+                    monitor.new_connection_monitor(endpoint),
+                )),
+                Err(e) => {
+                    monitor.on_err(endpoint);
+                    Err(e)
+                }
+            })
+    }
+}
+
+#[cfg(not(madsim))]
+impl<Con, E, C: futures::Stream<Item = Result<Con, E>>, M: MonitorNewConnection> futures::Stream
+    for MonitoredConnection<C, M>
+where
+    Con:
+        tonic::transport::server::Connected<ConnectInfo = tonic::transport::server::TcpConnectInfo>,
+{
+    type Item = Result<MonitoredConnection<Con, M::ConnectionMonitor>, E>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let (inner, monitor) = MonitoredConnection::project_into(self);
+        inner.poll_next(cx).map(|opt| {
+            opt.map(|result| {
+                result.map(|conn| {
+                    let remote_addr = conn.connect_info().remote_addr();
+                    let endpoint = remote_addr
+                        .map(|remote_addr| format!("{}:{}", remote_addr.ip(), remote_addr.port()))
+                        .unwrap_or("unknown".to_string());
+                    MonitoredConnection::new(conn, monitor.new_connection_monitor(endpoint))
+                })
+            })
+        })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
+#[derive(Clone)]
+pub struct ConnectionMetrics {
+    connection_count: GenericGaugeVec<AtomicI64>,
+    connection_create_rate: GenericCounterVec<AtomicU64>,
+    connection_err_rate: GenericCounterVec<AtomicU64>,
+
+    read_rate: GenericCounterVec<AtomicU64>,
+    reader_count: GenericGaugeVec<AtomicI64>,
+
+    write_rate: GenericCounterVec<AtomicU64>,
+    writer_count: GenericGaugeVec<AtomicI64>,
+
+    io_err_rate: GenericCounterVec<AtomicU64>,
+}
+
+pub static GLOBAL_CONNECTION_METRICS: LazyLock<ConnectionMetrics> =
+    LazyLock::new(|| ConnectionMetrics::new(&GLOBAL_METRICS_REGISTRY));
+
+impl ConnectionMetrics {
+    pub fn new(registry: &Registry) -> Self {
+        let labels = ["connection_type", "uri"];
+        let connection_count = register_int_gauge_vec_with_registry!(
+            "connection_count",
+            "The number of current existing connection",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let connection_create_rate = register_int_counter_vec_with_registry!(
+            "connection_create_rate",
+            "Rate on creating new connection",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let connection_err_rate = register_int_counter_vec_with_registry!(
+            "connection_err_rate",
+            "Error rate on creating new connection",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let read_rate = register_int_counter_vec_with_registry!(
+            "connection_read_rate",
+            "Read rate of a connection",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let reader_count = register_int_gauge_vec_with_registry!(
+            "connection_reader_count",
+            "The number of current existing reader",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let write_rate = register_int_counter_vec_with_registry!(
+            "connection_write_rate",
+            "Write rate of a connection",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let writer_count = register_int_gauge_vec_with_registry!(
+            "connection_writer_count",
+            "The number of current existing writer",
+            &labels,
+            registry,
+        )
+        .unwrap();
+
+        let io_err_rate = register_int_counter_vec_with_registry!(
+            "connection_io_err_rate",
+            "IO err rate of a connection",
+            &["connection_type", "uri", "op_type", "error_kind"],
+            registry,
+        )
+        .unwrap();
+
+        Self {
+            connection_count,
+            connection_create_rate,
+            connection_err_rate,
+            read_rate,
+            reader_count,
+            write_rate,
+            writer_count,
+            io_err_rate,
+        }
+    }
+}
+
+pub struct TcpConfig {
+    pub tcp_nodelay: bool,
+    pub keepalive_duration: Option<Duration>,
+}
+
+pub fn monitor_connector<C>(
+    connector: C,
+    connection_type: impl Into<String>,
+) -> MonitoredConnection<C, MonitorNewConnectionImpl> {
+    let connection_type = connection_type.into();
+    info!(
+        "monitoring connector {} with type {}",
+        type_name::<C>(),
+        connection_type
+    );
+    MonitoredConnection::new(connector, MonitorNewConnectionImpl { connection_type })
+}
+
+#[easy_ext::ext(EndpointExt)]
+impl Endpoint {
+    pub async fn monitored_connect(
+        self,
+        connection_type: impl Into<String>,
+        config: TcpConfig,
+    ) -> Result<Channel, tonic::transport::Error> {
+        #[cfg(not(madsim))]
+        {
+            let mut http = HttpConnector::new();
+            http.enforce_http(false);
+            http.set_nodelay(config.tcp_nodelay);
+            http.set_keepalive(config.keepalive_duration);
+
+            let connector = monitor_connector(http, connection_type);
+            self.connect_with_connector(connector).await
+        }
+        #[cfg(madsim)]
+        {
+            self.connect().await
+        }
+    }
+
+    #[cfg(not(madsim))]
+    pub fn monitored_connect_lazy(
+        self,
+        connection_type: impl Into<String>,
+        config: TcpConfig,
+    ) -> Channel {
+        let mut http = HttpConnector::new();
+        http.enforce_http(false);
+        http.set_nodelay(config.tcp_nodelay);
+        http.set_keepalive(config.keepalive_duration);
+
+        let connector = monitor_connector(http, connection_type);
+        self.connect_with_connector_lazy(connector)
+    }
+}
+
+#[cfg(not(madsim))]
+#[easy_ext::ext(RouterExt)]
+impl<L> tonic::transport::server::Router<L> {
+    pub async fn monitored_serve_with_shutdown<ResBody>(
+        self,
+        listen_addr: std::net::SocketAddr,
+        connection_type: impl Into<String>,
+        config: TcpConfig,
+        signal: impl Future<Output = ()>,
+    ) where
+        L: tower_layer::Layer<tonic::transport::server::Routes>,
+        L::Service: Service<
+                http::request::Request<hyper::Body>,
+                Response = http::response::Response<ResBody>,
+            > + Clone
+            + Send
+            + 'static,
+        <<L as tower_layer::Layer<tonic::transport::server::Routes>>::Service as Service<
+            http::request::Request<hyper::Body>,
+        >>::Future: Send + 'static,
+        <<L as tower_layer::Layer<tonic::transport::server::Routes>>::Service as Service<
+            http::request::Request<hyper::Body>,
+        >>::Error: Into<Box<dyn std::error::Error + Send + Sync>> + Send,
+        ResBody: http_body::Body<Data = bytes::Bytes> + Send + 'static,
+        ResBody::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        let incoming = tonic::transport::server::TcpIncoming::new(
+            listen_addr,
+            config.tcp_nodelay,
+            config.keepalive_duration,
+        )
+        .unwrap();
+        let incoming = MonitoredConnection::new(
+            incoming,
+            MonitorNewConnectionImpl {
+                connection_type: connection_type.into(),
+            },
+        );
+        self.serve_with_incoming_shutdown(incoming, signal)
+            .await
+            .unwrap()
+    }
+}
+
+#[cfg(madsim)]
+#[easy_ext::ext(RouterExt)]
+impl<L> tonic::transport::server::Router<L> {
+    pub async fn monitored_serve_with_shutdown(
+        self,
+        listen_addr: std::net::SocketAddr,
+        connection_type: impl Into<String>,
+        config: TcpConfig,
+        signal: impl Future<Output = ()>,
+    ) {
+        self.serve_with_shutdown(listen_addr, signal).await.unwrap()
+    }
+}
+
+#[cfg(not(madsim))]
+pub fn monitored_tcp_incoming(
+    listen_addr: std::net::SocketAddr,
+    connection_type: impl Into<String>,
+    config: TcpConfig,
+) -> Result<
+    MonitoredConnection<tonic::transport::server::TcpIncoming, MonitorNewConnectionImpl>,
+    Box<dyn std::error::Error + Send + Sync>,
+> {
+    let incoming = tonic::transport::server::TcpIncoming::new(
+        listen_addr,
+        config.tcp_nodelay,
+        config.keepalive_duration,
+    )?;
+    Ok(MonitoredConnection::new(
+        incoming,
+        MonitorNewConnectionImpl {
+            connection_type: connection_type.into(),
+        },
+    ))
+}
+
+#[derive(Clone)]
+pub struct MonitorNewConnectionImpl {
+    connection_type: String,
+}
+
+impl MonitorNewConnection for MonitorNewConnectionImpl {
+    type ConnectionMonitor = MonitorAsyncReadWriteImpl;
+
+    fn new_connection_monitor(&self, endpoint: String) -> Self::ConnectionMonitor {
+        let labels = [self.connection_type.as_str(), endpoint.as_str()];
+        let read_rate = GLOBAL_CONNECTION_METRICS
+            .read_rate
+            .with_label_values(&labels);
+        let reader_count = GLOBAL_CONNECTION_METRICS
+            .reader_count
+            .with_label_values(&labels);
+        let write_rate = GLOBAL_CONNECTION_METRICS
+            .write_rate
+            .with_label_values(&labels);
+        let writer_count = GLOBAL_CONNECTION_METRICS
+            .writer_count
+            .with_label_values(&labels);
+        let connection_count = GLOBAL_CONNECTION_METRICS
+            .connection_count
+            .with_label_values(&labels);
+
+        GLOBAL_CONNECTION_METRICS
+            .connection_create_rate
+            .with_label_values(&labels)
+            .inc();
+
+        MonitorAsyncReadWriteImpl::new(
+            endpoint,
+            self.connection_type.clone(),
+            read_rate,
+            reader_count,
+            write_rate,
+            writer_count,
+            connection_count,
+        )
+    }
+
+    fn on_err(&self, endpoint: String) {
+        GLOBAL_CONNECTION_METRICS
+            .connection_err_rate
+            .with_label_values(&[self.connection_type.as_str(), endpoint.as_str()])
+            .inc();
+    }
+}
+
+const READ_WRITE_RATE_REPORT_INTERVAL: u64 = 1024;
+
+pub struct MonitorAsyncReadWriteImpl {
+    endpoint: String,
+    connection_type: String,
+
+    unreported_read_rate: u64,
+    read_rate: GenericCounter<AtomicU64>,
+    reader_count_guard: GenericGauge<AtomicI64>,
+    is_eof: bool,
+
+    unreported_write_rate: u64,
+    write_rate: GenericCounter<AtomicU64>,
+    writer_count_guard: GenericGauge<AtomicI64>,
+    is_shutdown: bool,
+
+    connection_count_guard: GenericGauge<AtomicI64>,
+}
+
+impl MonitorAsyncReadWriteImpl {
+    pub fn new(
+        endpoint: String,
+        connection_type: String,
+        read_rate: GenericCounter<AtomicU64>,
+        reader_count: GenericGauge<AtomicI64>,
+        write_rate: GenericCounter<AtomicU64>,
+        writer_count: GenericGauge<AtomicI64>,
+        connection_count: GenericGauge<AtomicI64>,
+    ) -> Self {
+        reader_count.inc();
+        writer_count.inc();
+        connection_count.inc();
+        Self {
+            endpoint,
+            connection_type,
+            unreported_read_rate: 0,
+            read_rate,
+            reader_count_guard: reader_count,
+            is_eof: false,
+            unreported_write_rate: 0,
+            write_rate,
+            writer_count_guard: writer_count,
+            is_shutdown: false,
+            connection_count_guard: connection_count,
+        }
+    }
+}
+
+impl Drop for MonitorAsyncReadWriteImpl {
+    fn drop(&mut self) {
+        if self.unreported_read_rate > 0 {
+            self.read_rate.inc_by(self.unreported_read_rate);
+        }
+        if self.unreported_write_rate > 0 {
+            self.write_rate.inc_by(self.unreported_write_rate);
+        }
+        if !self.is_eof {
+            self.reader_count_guard.dec();
+        }
+        if !self.is_shutdown {
+            self.writer_count_guard.dec();
+        }
+        self.connection_count_guard.dec();
+    }
+}
+
+impl MonitorAsyncReadWrite for MonitorAsyncReadWriteImpl {
+    fn on_read(&mut self, size: usize) {
+        self.unreported_read_rate += size as u64;
+        if self.unreported_read_rate >= READ_WRITE_RATE_REPORT_INTERVAL {
+            self.read_rate.inc_by(self.unreported_read_rate);
+            self.unreported_read_rate = 0;
+        }
+    }
+
+    fn on_eof(&mut self) {
+        if self.is_eof {
+            warn!("get eof for multiple time");
+            return;
+        }
+        self.is_eof = true;
+        self.reader_count_guard.dec();
+    }
+
+    fn on_read_err(&mut self, err: &Error) {
+        GLOBAL_CONNECTION_METRICS
+            .io_err_rate
+            .with_label_values(&[
+                self.connection_type.as_str(),
+                self.endpoint.as_str(),
+                "read",
+                err.kind().to_string().as_str(),
+            ])
+            .inc();
+    }
+
+    fn on_write(&mut self, size: usize) {
+        self.unreported_write_rate += size as u64;
+        if self.unreported_write_rate >= READ_WRITE_RATE_REPORT_INTERVAL {
+            self.write_rate.inc_by(self.unreported_write_rate);
+            self.unreported_write_rate = 0;
+        }
+    }
+
+    fn on_shutdown(&mut self) {
+        if self.is_shutdown {
+            warn!("get shutdown for multiple time");
+            return;
+        }
+        self.is_shutdown = true;
+        self.writer_count_guard.dec();
+    }
+
+    fn on_write_err(&mut self, err: &Error) {
+        GLOBAL_CONNECTION_METRICS
+            .io_err_rate
+            .with_label_values(&[
+                self.connection_type.as_str(),
+                self.endpoint.as_str(),
+                "write",
+                err.kind().to_string().as_str(),
+            ])
+            .inc();
+    }
+}
diff --git a/src/common/src/monitor/mod.rs b/src/common/src/monitor/mod.rs
index b8339f58467ff..6c880f3720ad2 100644
--- a/src/common/src/monitor/mod.rs
+++ b/src/common/src/monitor/mod.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod connection;
 pub mod my_stats;
 pub mod process_linux;
 pub mod rwlock;
diff --git a/src/common/src/row/mod.rs b/src/common/src/row/mod.rs
index 572ad14edac07..691893587dd24 100644
--- a/src/common/src/row/mod.rs
+++ b/src/common/src/row/mod.rs
@@ -124,7 +124,16 @@ pub trait Row: Sized + std::fmt::Debug + PartialEq + Eq {
     /// Determines whether the datums of this row are equal to those of another.
     #[inline]
     fn eq(this: &Self, other: impl Row) -> bool {
-        this.iter().eq(other.iter())
+        if this.len() != other.len() {
+            return false;
+        }
+        for i in (0..this.len()).rev() {
+            // compare from the end to the start, as it's more likely to have same prefix
+            if this.datum_at(i) != other.datum_at(i) {
+                return false;
+            }
+        }
+        true
     }
 }
 
diff --git a/src/common/src/row/owned_row.rs b/src/common/src/row/owned_row.rs
index 2978c8ebd1a81..df5e9515f653b 100644
--- a/src/common/src/row/owned_row.rs
+++ b/src/common/src/row/owned_row.rs
@@ -45,7 +45,7 @@ impl AsRef<OwnedRow> for OwnedRow {
 impl OwnedRow {
     /// Returns an empty row.
     ///
-    /// Note: use [`empty`](super::empty) if possible.
+    /// Note: use [`empty`](super::empty()) if possible.
     pub fn empty() -> Self {
         Self(Box::new([]))
     }
diff --git a/src/common/src/row/project.rs b/src/common/src/row/project.rs
index 06a2bc6fe5f1a..ab4be6d229243 100644
--- a/src/common/src/row/project.rs
+++ b/src/common/src/row/project.rs
@@ -60,9 +60,10 @@ impl<'i, R: Row> Project<'i, R> {
     pub(crate) fn new(row: R, indices: &'i [usize]) -> Self {
         if let Some(index) = indices.iter().find(|&&i| i >= row.len()) {
             panic!(
-                "index {} out of bounds for row of length {}",
+                "index {} out of bounds for row of length {}, row {:?}",
                 index,
-                row.len()
+                row.len(),
+                row
             );
         }
         Self { row, indices }
diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs
index 0ccf6628898ee..7e8ad82665bab 100644
--- a/src/common/src/session_config/mod.rs
+++ b/src/common/src/session_config/mod.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod over_window;
 mod query_mode;
 mod search_path;
 mod transaction_isolation_level;
@@ -23,6 +24,7 @@ use std::ops::Deref;
 use chrono_tz::Tz;
 use educe::{self, Educe};
 use itertools::Itertools;
+pub use over_window::OverWindowCachePolicy;
 pub use query_mode::QueryMode;
 pub use search_path::{SearchPath, USER_NAME_WILD_CARD};
 use tracing::info;
@@ -34,7 +36,7 @@ use crate::util::epoch::Epoch;
 
 // This is a hack, &'static str is not allowed as a const generics argument.
 // TODO: refine this using the adt_const_params feature.
-const CONFIG_KEYS: [&str; 34] = [
+const CONFIG_KEYS: [&str; 37] = [
     "RW_IMPLICIT_FLUSH",
     "CREATE_COMPACTION_GROUP_FOR_MV",
     "QUERY_MODE",
@@ -69,6 +71,9 @@ const CONFIG_KEYS: [&str; 34] = [
     "LOCK_TIMEOUT",
     "ROW_SECURITY",
     "STANDARD_CONFORMING_STRINGS",
+    "RW_STREAMING_RATE_LIMIT",
+    "CDC_BACKFILL",
+    "RW_STREAMING_OVER_WINDOW_CACHE_POLICY",
 ];
 
 // MUST HAVE 1v1 relationship to CONFIG_KEYS. e.g. CONFIG_KEYS[IMPLICIT_FLUSH] =
@@ -107,6 +112,9 @@ const STATEMENT_TIMEOUT: usize = 30;
 const LOCK_TIMEOUT: usize = 31;
 const ROW_SECURITY: usize = 32;
 const STANDARD_CONFORMING_STRINGS: usize = 33;
+const RW_STREAMING_RATE_LIMIT: usize = 34;
+const CDC_BACKFILL: usize = 35;
+const STREAMING_OVER_WINDOW_CACHE_POLICY: usize = 36;
 
 trait ConfigEntry: Default + for<'a> TryFrom<&'a [&'a str], Error = RwError> {
     fn entry_name() -> &'static str;
@@ -329,6 +337,8 @@ type StatementTimeout = ConfigI32<STATEMENT_TIMEOUT, 0>;
 type LockTimeout = ConfigI32<LOCK_TIMEOUT, 0>;
 type RowSecurity = ConfigBool<ROW_SECURITY, true>;
 type StandardConformingStrings = ConfigString<STANDARD_CONFORMING_STRINGS>;
+type StreamingRateLimit = ConfigU64<RW_STREAMING_RATE_LIMIT, 0>;
+type CdcBackfill = ConfigBool<CDC_BACKFILL, false>;
 
 /// Report status or notice to caller.
 pub trait ConfigReporter {
@@ -468,6 +478,14 @@ pub struct ConfigMap {
         expression = "ConfigString::<STANDARD_CONFORMING_STRINGS>(String::from(\"on\"))"
     ))]
     standard_conforming_strings: StandardConformingStrings,
+
+    streaming_rate_limit: StreamingRateLimit,
+
+    cdc_backfill: CdcBackfill,
+
+    /// Cache policy for partition cache in streaming over window.
+    /// Can be "full", "recent", "recent_first_n" or "recent_last_n".
+    streaming_over_window_cache_policy: OverWindowCachePolicy,
 }
 
 impl ConfigMap {
@@ -579,6 +597,12 @@ impl ConfigMap {
             self.row_security = val.as_slice().try_into()?;
         } else if key.eq_ignore_ascii_case(StandardConformingStrings::entry_name()) {
             self.standard_conforming_strings = val.as_slice().try_into()?;
+        } else if key.eq_ignore_ascii_case(StreamingRateLimit::entry_name()) {
+            self.streaming_rate_limit = val.as_slice().try_into()?;
+        } else if key.eq_ignore_ascii_case(CdcBackfill::entry_name()) {
+            self.cdc_backfill = val.as_slice().try_into()?
+        } else if key.eq_ignore_ascii_case(OverWindowCachePolicy::entry_name()) {
+            self.streaming_over_window_cache_policy = val.as_slice().try_into()?;
         } else {
             return Err(ErrorCode::UnrecognizedConfigurationParameter(key.to_string()).into());
         }
@@ -660,6 +684,12 @@ impl ConfigMap {
             Ok(self.row_security.to_string())
         } else if key.eq_ignore_ascii_case(StandardConformingStrings::entry_name()) {
             Ok(self.standard_conforming_strings.to_string())
+        } else if key.eq_ignore_ascii_case(StreamingRateLimit::entry_name()) {
+            Ok(self.streaming_rate_limit.to_string())
+        } else if key.eq_ignore_ascii_case(CdcBackfill::entry_name()) {
+            Ok(self.cdc_backfill.to_string())
+        } else if key.eq_ignore_ascii_case(OverWindowCachePolicy::entry_name()) {
+            Ok(self.streaming_over_window_cache_policy.to_string())
         } else {
             Err(ErrorCode::UnrecognizedConfigurationParameter(key.to_string()).into())
         }
@@ -836,7 +866,22 @@ impl ConfigMap {
                 name: StandardConformingStrings::entry_name().to_lowercase(),
                 setting: self.standard_conforming_strings.to_string(),
                 description: String::from("Unused in RisingWave"),
-            }
+            },
+            VariableInfo{
+                name: StreamingRateLimit::entry_name().to_lowercase(),
+                setting: self.streaming_rate_limit.to_string(),
+                description: String::from("Set streaming rate limit (rows per second) for each parallelism for mv backfilling"),
+            },
+            VariableInfo{
+                name: CdcBackfill::entry_name().to_lowercase(),
+                setting: self.cdc_backfill.to_string(),
+                description: String::from("Enable backfill for CDC table to allow lock-free and incremental snapshot"),
+            },
+            VariableInfo{
+                name: OverWindowCachePolicy::entry_name().to_lowercase(),
+                setting: self.streaming_over_window_cache_policy.to_string(),
+                description: String::from(r#"Cache policy for partition cache in streaming over window. Can be "full", "recent", "recent_first_n" or "recent_last_n"."#),
+            },
         ]
     }
 
@@ -960,4 +1005,19 @@ impl ConfigMap {
     pub fn get_standard_conforming_strings(&self) -> &str {
         &self.standard_conforming_strings
     }
+
+    pub fn get_streaming_rate_limit(&self) -> Option<u32> {
+        if self.streaming_rate_limit.0 != 0 {
+            return Some(self.streaming_rate_limit.0 as u32);
+        }
+        None
+    }
+
+    pub fn get_cdc_backfill(&self) -> bool {
+        self.cdc_backfill.0
+    }
+
+    pub fn get_streaming_over_window_cache_policy(&self) -> OverWindowCachePolicy {
+        self.streaming_over_window_cache_policy
+    }
 }
diff --git a/src/common/src/session_config/over_window.rs b/src/common/src/session_config/over_window.rs
new file mode 100644
index 0000000000000..832d7a1e6c288
--- /dev/null
+++ b/src/common/src/session_config/over_window.rs
@@ -0,0 +1,125 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::str::FromStr;
+
+use enum_as_inner::EnumAsInner;
+use parse_display::{Display, FromStr};
+use risingwave_pb::stream_plan::PbOverWindowCachePolicy;
+
+use super::{ConfigEntry, CONFIG_KEYS, STREAMING_OVER_WINDOW_CACHE_POLICY};
+use crate::error::ErrorCode::{self, InvalidConfigValue};
+use crate::error::RwError;
+
+#[derive(Copy, Default, Debug, Clone, PartialEq, Eq, FromStr, Display, EnumAsInner)]
+#[display(style = "snake_case")]
+pub enum OverWindowCachePolicy {
+    /// Cache all entries.
+    #[default]
+    Full,
+    /// Cache only recently accessed range of entries.
+    Recent,
+    /// Cache only the first N entries in recently accessed range.
+    RecentFirstN,
+    /// Cache only the last N entries in recently accessed range.
+    RecentLastN,
+}
+
+impl TryFrom<&[&str]> for OverWindowCachePolicy {
+    type Error = RwError;
+
+    fn try_from(value: &[&str]) -> Result<Self, Self::Error> {
+        if value.len() != 1 {
+            return Err(ErrorCode::InternalError(format!(
+                "SET {} takes only one argument",
+                Self::entry_name()
+            ))
+            .into());
+        }
+
+        let s = value[0].to_ascii_lowercase().replace('-', "_");
+        OverWindowCachePolicy::from_str(&s).map_err(|_| {
+            InvalidConfigValue {
+                config_entry: Self::entry_name().to_string(),
+                config_value: s.to_string(),
+            }
+            .into()
+        })
+    }
+}
+
+impl ConfigEntry for OverWindowCachePolicy {
+    fn entry_name() -> &'static str {
+        CONFIG_KEYS[STREAMING_OVER_WINDOW_CACHE_POLICY]
+    }
+}
+
+impl OverWindowCachePolicy {
+    pub fn to_protobuf(self) -> PbOverWindowCachePolicy {
+        match self {
+            Self::Full => PbOverWindowCachePolicy::Full,
+            Self::Recent => PbOverWindowCachePolicy::Recent,
+            Self::RecentFirstN => PbOverWindowCachePolicy::RecentFirstN,
+            Self::RecentLastN => PbOverWindowCachePolicy::RecentLastN,
+        }
+    }
+
+    pub fn from_protobuf(pb: PbOverWindowCachePolicy) -> Self {
+        match pb {
+            PbOverWindowCachePolicy::Unspecified => Self::default(),
+            PbOverWindowCachePolicy::Full => Self::Full,
+            PbOverWindowCachePolicy::Recent => Self::Recent,
+            PbOverWindowCachePolicy::RecentFirstN => Self::RecentFirstN,
+            PbOverWindowCachePolicy::RecentLastN => Self::RecentLastN,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_over_window_cache_policy() {
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["full"].as_slice()).unwrap(),
+            OverWindowCachePolicy::Full
+        );
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["recent"].as_slice()).unwrap(),
+            OverWindowCachePolicy::Recent
+        );
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["RECENT"].as_slice()).unwrap(),
+            OverWindowCachePolicy::Recent
+        );
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["recent_first_n"].as_slice()).unwrap(),
+            OverWindowCachePolicy::RecentFirstN
+        );
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["recent_last_n"].as_slice()).unwrap(),
+            OverWindowCachePolicy::RecentLastN
+        );
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["recent-last-n"].as_slice()).unwrap(),
+            OverWindowCachePolicy::RecentLastN
+        );
+        assert_eq!(
+            OverWindowCachePolicy::try_from(["recent_last_N"].as_slice()).unwrap(),
+            OverWindowCachePolicy::RecentLastN
+        );
+        assert!(OverWindowCachePolicy::try_from(["foo"].as_slice()).is_err());
+    }
+}
diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs
index f2503fa7325fe..ccb666367d729 100644
--- a/src/common/src/system_param/mod.rs
+++ b/src/common/src/system_param/mod.rs
@@ -17,7 +17,7 @@
 //!
 //! To add a new system parameter:
 //! - Add a new field to [`PbSystemParams`] in `meta.proto`.
-//! - Add a new entry to [`for_all_undeprecated_params`] in this file.
+//! - Add a new entry to `for_all_undeprecated_params` in this file.
 //! - Add a new method to [`reader::SystemParamsReader`].
 
 pub mod local_manager;
@@ -56,7 +56,8 @@ macro_rules! for_all_undeprecated_params {
             { backup_storage_directory, String, Some("backup".to_string()), false },
             { telemetry_enabled, bool, Some(true), true },
             { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true },
-            $({ $field, $type, $default },)*
+            { pause_on_next_bootstrap, bool, Some(false), true },
+            $({ $field, $type, $default, $is_mutable },)*
         }
     };
 }
@@ -67,7 +68,7 @@ macro_rules! for_all_params {
     ($macro:ident) => {
         for_all_undeprecated_params!(
             $macro /* Define future deprecated params here, such as
-                    * ,{ backup_storage_directory, String, "backup".to_string() } */
+                    * ,{ backup_storage_directory, String, "backup".to_string(), true } */
         );
     };
 }
@@ -370,6 +371,7 @@ mod tests {
             (BACKUP_STORAGE_DIRECTORY_KEY, "a"),
             (TELEMETRY_ENABLED_KEY, "false"),
             (MAX_CONCURRENT_CREATING_STREAMING_JOBS_KEY, "1"),
+            (PAUSE_ON_NEXT_BOOTSTRAP_KEY, "false"),
         ];
 
         // To kv - missing field.
diff --git a/src/common/src/system_param/reader.rs b/src/common/src/system_param/reader.rs
index 63954d53f9d52..91cc893823b66 100644
--- a/src/common/src/system_param/reader.rs
+++ b/src/common/src/system_param/reader.rs
@@ -80,6 +80,10 @@ impl SystemParamsReader {
         self.prost.telemetry_enabled.unwrap()
     }
 
+    pub fn pause_on_next_bootstrap(&self) -> bool {
+        self.prost.pause_on_next_bootstrap.unwrap_or(false)
+    }
+
     pub fn to_kv(&self) -> Vec<(String, String)> {
         system_params_to_kv(&self.prost).unwrap()
     }
diff --git a/src/common/src/types/interval.rs b/src/common/src/types/interval.rs
index c2bed9770b13b..aca4d090bcac2 100644
--- a/src/common/src/types/interval.rs
+++ b/src/common/src/types/interval.rs
@@ -1025,7 +1025,7 @@ impl Interval {
     /// Converts str to interval
     ///
     /// The input str must have the following format:
-    /// P<years>Y<months>M<days>DT<hours>H<minutes>M<seconds>S
+    /// `P<years>Y<months>M<days>DT<hours>H<minutes>M<seconds>S`
     ///
     /// Example
     /// - P1Y2M3DT4H5M6.78S
diff --git a/src/common/src/types/jsonb.rs b/src/common/src/types/jsonb.rs
index d576876cb0b66..2d08347014548 100644
--- a/src/common/src/types/jsonb.rs
+++ b/src/common/src/types/jsonb.rs
@@ -283,10 +283,11 @@ impl<'a> JsonbRef<'a> {
     /// This is part of the `->>` or `#>>` syntax to access a child as string.
     ///
     /// * It is not `as_str`, because there is no runtime error when the jsonb type is not string.
-    /// * It is not same as [`Display`] or [`ToText`] (cast to string) in the following 2 cases:
+    /// * It is not same as [`std::fmt::Display`] or [`super::ToText`] (cast to string) in the
+    ///   following 2 cases:
     ///   * Jsonb null is displayed as 4-letter `null` but treated as sql null here.
     ///       * This function writes nothing and the caller is responsible for checking
-    ///         [`is_jsonb_null`] to differentiate it from an empty string.
+    ///         [`Self::is_jsonb_null`] to differentiate it from an empty string.
     ///   * Jsonb string is displayed with quotes but treated as its inner value here.
     pub fn force_str<W: std::fmt::Write>(&self, writer: &mut W) -> std::fmt::Result {
         match self.0 {
diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs
index 7737d76cd48fc..d0e155f784546 100644
--- a/src/common/src/types/mod.rs
+++ b/src/common/src/types/mod.rs
@@ -375,6 +375,18 @@ impl DataType {
         }
     }
 
+    /// Returns the inner type of a list type.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the type is not a list type.
+    pub fn as_list(&self) -> &DataType {
+        match self {
+            DataType::List(t) => t,
+            _ => panic!("expect list type"),
+        }
+    }
+
     /// WARNING: Currently this should only be used in `WatermarkFilterExecutor`. Please be careful
     /// if you want to use this.
     pub fn min_value(&self) -> ScalarImpl {
diff --git a/src/common/src/types/timestamptz.rs b/src/common/src/types/timestamptz.rs
index 0d9af9a5e3d3f..1f9b962c9d376 100644
--- a/src/common/src/types/timestamptz.rs
+++ b/src/common/src/types/timestamptz.rs
@@ -16,7 +16,7 @@ use std::io::Write;
 use std::str::FromStr;
 
 use bytes::{Bytes, BytesMut};
-use chrono::{DateTime, TimeZone, Utc};
+use chrono::{TimeZone, Utc};
 use chrono_tz::Tz;
 use postgres_types::ToSql;
 use serde::{Deserialize, Serialize};
@@ -148,8 +148,32 @@ impl FromStr for Timestamptz {
             "Can't cast string to timestamp with time zone (expected format is YYYY-MM-DD HH:MM:SS[.D+{up to 6 digits}] followed by +hh:mm or literal Z)"
             , "\nFor example: '2021-04-01 00:00:00+00:00'"
         );
-        let ret = s.parse::<DateTime<Utc>>().map_err(|_| ERROR_MSG)?;
-        Ok(Timestamptz(ret.timestamp_micros()))
+        // Try `speedate` first
+        // * It is also used by `str_to_{date,time,timestamp}`
+        // * It can parse without seconds `2006-01-02 15:04-07:00`
+        let ret = match speedate::DateTime::parse_str_rfc3339(s) {
+            Ok(r) => r,
+            Err(_) => {
+                // Supplement with `chrono` for existing cases:
+                // * Extra space before offset `2006-01-02 15:04:05 -07:00`
+                return s
+                    .parse::<chrono::DateTime<Utc>>()
+                    .map(|t| Timestamptz(t.timestamp_micros()))
+                    .map_err(|_| ERROR_MSG);
+            }
+        };
+        if ret.time.tz_offset.is_none() {
+            return Err(ERROR_MSG);
+        }
+        if ret.date.year < 1600 {
+            return Err("parsing timestamptz with year < 1600 unsupported");
+        }
+        Ok(Timestamptz(
+            ret.timestamp_tz()
+                .checked_mul(1000000)
+                .and_then(|us| us.checked_add(ret.time.microsecond.into()))
+                .ok_or(ERROR_MSG)?,
+        ))
     }
 }
 
diff --git a/src/common/src/util/chunk_coalesce.rs b/src/common/src/util/chunk_coalesce.rs
index 0a8d256a92de9..3c5e4b109da90 100644
--- a/src/common/src/util/chunk_coalesce.rs
+++ b/src/common/src/util/chunk_coalesce.rs
@@ -165,14 +165,7 @@ impl DataChunkBuilder {
         }
     }
 
-    /// Append one row from the given [`Row`].
-    /// The caller is responsible to build the data chunk,
-    /// i.e. call the `build_data_chunk` function.
-    /// Used when caller wants to decide when to `build_data_chunk`.
-    /// For instance, in backfill, we want to build data chunk when:
-    /// 1. Buffer is full (typical case)
-    /// 2. On barrier, to flush the remaining data in `data_chunk_builder`.
-    pub fn append_one_row_no_finish(&mut self, row: impl Row) {
+    fn append_one_row_no_finish(&mut self, row: impl Row) {
         assert!(self.buffered_count < self.batch_size);
         self.ensure_builders();
         self.do_append_one_row_from_datums(row.iter());
@@ -212,7 +205,7 @@ impl DataChunkBuilder {
         }
     }
 
-    pub fn build_data_chunk(&mut self) -> DataChunk {
+    fn build_data_chunk(&mut self) -> DataChunk {
         let mut finished_array_builders = vec![];
         swap(&mut finished_array_builders, &mut self.array_builders);
         let cardinality = self.buffered_count;
@@ -236,6 +229,17 @@ impl DataChunkBuilder {
     pub fn data_types(&self) -> Vec<DataType> {
         self.data_types.clone()
     }
+
+    pub fn is_empty(&self) -> bool {
+        self.buffered_count == 0
+    }
+
+    pub fn clear(&mut self) {
+        if !self.is_empty() {
+            self.array_builders.clear()
+        }
+        self.buffered_count = 0;
+    }
 }
 
 impl Drop for DataChunkBuilder {
diff --git a/src/common/src/util/future_utils.rs b/src/common/src/util/future_utils.rs
index 0e27d484a8bac..4b077ef76fdbd 100644
--- a/src/common/src/util/future_utils.rs
+++ b/src/common/src/util/future_utils.rs
@@ -26,9 +26,10 @@ pub fn select_all<S: Stream + Unpin>(
 }
 
 pub fn pending_on_none<I>(future: impl Future<Output = Option<I>>) -> impl Future<Output = I> {
-    use futures::TryFutureExt;
-    future
-        .map(|opt| opt.ok_or(()))
-        .or_else(|()| pending::<std::result::Result<I, ()>>())
-        .map(|result| result.expect("only err on pending, which is unlikely to reach here"))
+    future.then(|opt| async move {
+        match opt {
+            Some(item) => item,
+            None => pending::<I>().await,
+        }
+    })
 }
diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs
index 363099ec35f9f..e19bdaeb1427a 100644
--- a/src/compute/src/lib.rs
+++ b/src/compute/src/lib.rs
@@ -32,8 +32,11 @@ pub mod rpc;
 pub mod server;
 pub mod telemetry;
 
+use std::future::Future;
+use std::pin::Pin;
+
 use clap::{Parser, ValueEnum};
-use risingwave_common::config::{AsyncStackTraceOption, OverrideConfig};
+use risingwave_common::config::{AsyncStackTraceOption, MetricLevel, OverrideConfig};
 use risingwave_common::util::resource_util::cpu::total_cpu_available;
 use risingwave_common::util::resource_util::memory::total_memory_available_bytes;
 use serde::{Deserialize, Serialize};
@@ -100,7 +103,7 @@ pub struct ComputeNodeOpts {
     /// >0 = enable metrics
     #[clap(long, env = "RW_METRICS_LEVEL")]
     #[override_opts(path = server.metrics_level)]
-    pub metrics_level: Option<u32>,
+    pub metrics_level: Option<MetricLevel>,
 
     /// Path to data file cache data directory.
     /// Left empty to disable file cache.
@@ -186,9 +189,6 @@ fn validate_opts(opts: &ComputeNodeOpts) {
     }
 }
 
-use std::future::Future;
-use std::pin::Pin;
-
 use crate::server::compute_node_serve;
 
 /// Start compute node
diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs
index 8bfe69effb011..46340f9f4c75d 100644
--- a/src/compute/src/server.rs
+++ b/src/compute/src/server.rs
@@ -23,9 +23,10 @@ use risingwave_batch::monitor::{
 use risingwave_batch::rpc::service::task_service::BatchServiceImpl;
 use risingwave_batch::task::{BatchEnvironment, BatchManager};
 use risingwave_common::config::{
-    load_config, AsyncStackTraceOption, StorageMemoryConfig, MAX_CONNECTION_WINDOW_SIZE,
-    STREAM_WINDOW_SIZE,
+    load_config, AsyncStackTraceOption, MetricLevel, StorageMemoryConfig,
+    MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE,
 };
+use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
 use risingwave_common::telemetry::manager::TelemetryManager;
 use risingwave_common::telemetry::telemetry_env_enabled;
@@ -53,12 +54,12 @@ use risingwave_storage::hummock::compactor::{
 use risingwave_storage::hummock::hummock_meta_client::MonitoredHummockMetaClient;
 use risingwave_storage::hummock::{HummockMemoryCollector, MemoryLimiter};
 use risingwave_storage::monitor::{
-    monitor_cache, GLOBAL_COMPACTOR_METRICS, GLOBAL_HUMMOCK_METRICS,
-    GLOBAL_HUMMOCK_STATE_STORE_METRICS, GLOBAL_OBJECT_STORE_METRICS, GLOBAL_STORAGE_METRICS,
+    global_hummock_state_store_metrics, global_storage_metrics, monitor_cache,
+    GLOBAL_COMPACTOR_METRICS, GLOBAL_HUMMOCK_METRICS, GLOBAL_OBJECT_STORE_METRICS,
 };
 use risingwave_storage::opts::StorageOpts;
 use risingwave_storage::StateStoreImpl;
-use risingwave_stream::executor::monitor::GLOBAL_STREAMING_METRICS;
+use risingwave_stream::executor::monitor::global_streaming_metrics;
 use risingwave_stream::task::{LocalStreamManager, StreamEnvironment};
 use tokio::sync::oneshot::Sender;
 use tokio::task::JoinHandle;
@@ -168,16 +169,18 @@ pub async fn compute_node_serve(
     // Initialize the metrics subsystem.
     let source_metrics = Arc::new(GLOBAL_SOURCE_METRICS.clone());
     let hummock_metrics = Arc::new(GLOBAL_HUMMOCK_METRICS.clone());
-    let streaming_metrics = Arc::new(GLOBAL_STREAMING_METRICS.clone());
+    let streaming_metrics = Arc::new(global_streaming_metrics(config.server.metrics_level));
     let batch_task_metrics = Arc::new(GLOBAL_BATCH_TASK_METRICS.clone());
     let batch_executor_metrics = Arc::new(GLOBAL_BATCH_EXECUTOR_METRICS.clone());
     let batch_manager_metrics = GLOBAL_BATCH_MANAGER_METRICS.clone();
     let exchange_srv_metrics = Arc::new(GLOBAL_EXCHANGE_SERVICE_METRICS.clone());
 
     // Initialize state store.
-    let state_store_metrics = Arc::new(GLOBAL_HUMMOCK_STATE_STORE_METRICS.clone());
+    let state_store_metrics = Arc::new(global_hummock_state_store_metrics(
+        config.server.metrics_level,
+    ));
     let object_store_metrics = Arc::new(GLOBAL_OBJECT_STORE_METRICS.clone());
-    let storage_metrics = Arc::new(GLOBAL_STORAGE_METRICS.clone());
+    let storage_metrics = Arc::new(global_storage_metrics(config.server.metrics_level));
     let compactor_metrics = Arc::new(GLOBAL_COMPACTOR_METRICS.clone());
     let hummock_meta_client = Arc::new(MonitoredHummockMetaClient::new(
         meta_client.clone(),
@@ -213,22 +216,25 @@ pub async fn compute_node_serve(
             let memory_limiter = Arc::new(MemoryLimiter::new(
                 storage_opts.compactor_memory_limit_mb as u64 * 1024 * 1024 / 2,
             ));
-            let compactor_context = Arc::new(CompactorContext {
+            let compactor_context = CompactorContext {
                 storage_opts,
-                hummock_meta_client: hummock_meta_client.clone(),
                 sstable_store: storage.sstable_store(),
                 compactor_metrics: compactor_metrics.clone(),
                 is_share_buffer_compact: false,
                 compaction_executor: Arc::new(CompactionExecutor::new(Some(1))),
                 filter_key_extractor_manager: storage.filter_key_extractor_manager().clone(),
                 memory_limiter,
-                sstable_object_id_manager: storage.sstable_object_id_manager().clone(),
+
                 task_progress_manager: Default::default(),
                 await_tree_reg: None,
                 running_task_count: Arc::new(AtomicU32::new(0)),
-            });
+            };
 
-            let (handle, shutdown_sender) = start_compactor(compactor_context);
+            let (handle, shutdown_sender) = start_compactor(
+                compactor_context,
+                hummock_meta_client.clone(),
+                storage.sstable_object_id_manager().clone(),
+            );
             sub_tasks.push((handle, shutdown_sender));
         }
         let flush_limiter = storage.get_memory_limiter();
@@ -396,7 +402,6 @@ pub async fn compute_node_serve(
         tonic::transport::Server::builder()
             .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE)
             .initial_stream_window_size(STREAM_WINDOW_SIZE)
-            .tcp_nodelay(true)
             .layer(TracingExtractLayer::new())
             // XXX: unlimit the max message size to allow arbitrary large SQL input.
             .add_service(TaskServiceServer::new(batch_srv).max_decoding_message_size(usize::MAX))
@@ -418,29 +423,36 @@ pub async fn compute_node_serve(
             .add_service(MonitorServiceServer::new(monitor_srv))
             .add_service(ConfigServiceServer::new(config_srv))
             .add_service(HealthServer::new(health_srv))
-            .serve_with_shutdown(listen_addr, async move {
-                tokio::select! {
-                    _ = tokio::signal::ctrl_c() => {},
-                    _ = &mut shutdown_recv => {
-                        for (join_handle, shutdown_sender) in sub_tasks {
-                            if let Err(err) = shutdown_sender.send(()) {
-                                tracing::warn!("Failed to send shutdown: {:?}", err);
-                                continue;
-                            }
-                            if let Err(err) = join_handle.await {
-                                tracing::warn!("Failed to join shutdown: {:?}", err);
+            .monitored_serve_with_shutdown(
+                listen_addr,
+                "grpc-compute-node-service",
+                TcpConfig {
+                    tcp_nodelay: true,
+                    keepalive_duration: None,
+                },
+                async move {
+                    tokio::select! {
+                        _ = tokio::signal::ctrl_c() => {},
+                        _ = &mut shutdown_recv => {
+                            for (join_handle, shutdown_sender) in sub_tasks {
+                                if let Err(err) = shutdown_sender.send(()) {
+                                    tracing::warn!("Failed to send shutdown: {:?}", err);
+                                    continue;
+                                }
+                                if let Err(err) = join_handle.await {
+                                    tracing::warn!("Failed to join shutdown: {:?}", err);
+                                }
                             }
-                        }
-                    },
-                }
-            })
-            .await
-            .unwrap();
+                        },
+                    }
+                },
+            )
+            .await;
     });
     join_handle_vec.push(join_handle);
 
     // Boot metrics service.
-    if config.server.metrics_level > 0 {
+    if config.server.metrics_level > MetricLevel::Disabled {
         MetricsManager::boot_metrics_service(opts.prometheus_listener_addr.clone());
     }
 
diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs
new file mode 100644
index 0000000000000..90d8f0c3a5541
--- /dev/null
+++ b/src/compute/tests/cdc_tests.rs
@@ -0,0 +1,333 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(let_chains)]
+#![feature(generators)]
+
+use std::sync::atomic::AtomicU64;
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures::stream::StreamExt;
+use futures_async_stream::try_stream;
+use itertools::Itertools;
+use risingwave_batch::executor::{Executor as BatchExecutor, RowSeqScanExecutor, ScanRange};
+use risingwave_common::array::{
+    Array, ArrayBuilder, StreamChunk, StreamChunkTestExt, Utf8ArrayBuilder,
+};
+use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, Field, Schema, TableId};
+use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
+use risingwave_connector::source::external::{
+    DebeziumOffset, DebeziumSourceOffset, ExternalTableReaderImpl, MySqlOffset, SchemaTableName,
+};
+use risingwave_connector::source::MockExternalTableReader;
+use risingwave_hummock_sdk::to_committed_batch_query_epoch;
+use risingwave_storage::memory::MemoryStateStore;
+use risingwave_storage::table::batch_table::storage_table::StorageTable;
+use risingwave_stream::error::StreamResult;
+use risingwave_stream::executor::external::ExternalStorageTable;
+use risingwave_stream::executor::monitor::StreamingMetrics;
+use risingwave_stream::executor::test_utils::MockSource;
+use risingwave_stream::executor::{
+    default_source_internal_table, expect_first_barrier, ActorContext, Barrier,
+    BoxedExecutor as StreamBoxedExecutor, BoxedMessageStream, CdcBackfillExecutor, Executor,
+    MaterializeExecutor, Message, Mutation, PkIndices, PkIndicesRef, SourceStateTableHandler,
+    StreamExecutorError,
+};
+
+// mock upstream binlog offset starting from "1.binlog, pos=0"
+pub struct MockOffsetGenExecutor {
+    upstream: Option<StreamBoxedExecutor>,
+
+    schema: Schema,
+
+    pk_indices: PkIndices,
+
+    identity: String,
+
+    start_offset: u32,
+}
+
+impl MockOffsetGenExecutor {
+    pub fn new(upstream: StreamBoxedExecutor, schema: Schema, pk_indices: PkIndices) -> Self {
+        Self {
+            upstream: Some(upstream),
+            schema,
+            pk_indices,
+            identity: "MockOffsetGenExecutor".to_string(),
+            start_offset: 0,
+        }
+    }
+
+    fn next_offset(&mut self) -> anyhow::Result<String> {
+        let start_offset = self.start_offset;
+        let dbz_offset = DebeziumOffset {
+            source_partition: Default::default(),
+            source_offset: DebeziumSourceOffset {
+                last_snapshot_record: None,
+                snapshot: None,
+                file: Some("1.binlog".to_string()),
+                pos: Some(start_offset as _),
+                lsn: None,
+                txid: None,
+                tx_usec: None,
+            },
+        };
+
+        self.start_offset += 1;
+        let out = serde_json::to_string(&dbz_offset)?;
+        Ok(out)
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn execute_inner(mut self) {
+        // fake offset in stream chunk
+        let mut upstream = self.upstream.take().unwrap().execute();
+
+        // The first barrier mush propagated.
+        let barrier = expect_first_barrier(&mut upstream).await?;
+        yield Message::Barrier(barrier);
+
+        #[for_await]
+        for msg in upstream {
+            let msg = msg?;
+
+            match msg {
+                Message::Chunk(chunk) => {
+                    let mut offset_builder = Utf8ArrayBuilder::new(chunk.cardinality());
+                    let (ops, mut columns, vis) = chunk.into_inner();
+                    assert!(vis.as_visibility().is_none());
+
+                    for _ in 0..ops.len() {
+                        let offset_str = self.next_offset()?;
+                        offset_builder.append(Some(&offset_str));
+                    }
+
+                    let offsets = offset_builder.finish();
+                    columns.push(offsets.into_ref());
+                    yield Message::Chunk(StreamChunk::new(ops, columns, vis.into_visibility()));
+                }
+                Message::Barrier(barrier) => {
+                    yield Message::Barrier(barrier);
+                }
+                Message::Watermark(watermark) => yield Message::Watermark(watermark),
+            }
+        }
+    }
+}
+
+impl Executor for MockOffsetGenExecutor {
+    fn execute(self: Box<Self>) -> BoxedMessageStream {
+        self.execute_inner().boxed()
+    }
+
+    fn schema(&self) -> &Schema {
+        &self.schema
+    }
+
+    fn pk_indices(&self) -> PkIndicesRef<'_> {
+        &self.pk_indices
+    }
+
+    fn identity(&self) -> &str {
+        &self.identity
+    }
+}
+
+#[tokio::test]
+async fn test_cdc_backfill() -> StreamResult<()> {
+    use risingwave_common::types::DataType;
+    let memory_state_store = MemoryStateStore::new();
+
+    let table_id = TableId::new(1002);
+    let schema = Schema::new(vec![
+        Field::unnamed(DataType::Int64), // primary key
+        Field::unnamed(DataType::Float64),
+        Field::unnamed(DataType::Varchar),
+    ]);
+    let column_ids = vec![0.into(), 1.into(), 2.into()];
+
+    let pk_indices = vec![0];
+
+    let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone());
+    let _actor_ctx = ActorContext::create(0x3a3a3a);
+
+    // mock upstream offset (start from "1.binlog, pos=0") for ingested chunks
+    let mock_offset_executor =
+        MockOffsetGenExecutor::new(Box::new(source), schema.clone(), pk_indices.clone());
+
+    let binlog_file = String::from("1.binlog");
+
+    // mock binlog watermarks for backfill
+    // initial low watermark: 1.binlog, pos=2 and expected behaviors:
+    // - ignore events before (1.binlog, pos=2);
+    // - apply events in the range of (1.binlog, pos=2, 1.binlog, pos=4) to the snapshot
+    let binlog_watermarks = vec![
+        MySqlOffset::new(binlog_file.clone(), 2), // binlog low watermark
+        MySqlOffset::new(binlog_file.clone(), 4),
+        MySqlOffset::new(binlog_file.clone(), 6),
+        MySqlOffset::new(binlog_file.clone(), 8),
+        MySqlOffset::new(binlog_file.clone(), 10),
+    ];
+
+    let table_name = SchemaTableName::new("mock_table".to_string(), "public".to_string());
+    let external_table = ExternalStorageTable::new(
+        table_id,
+        table_name,
+        ExternalTableReaderImpl::Mock(MockExternalTableReader::new(binlog_watermarks)),
+        schema.clone(),
+        vec![OrderType::ascending()],
+        pk_indices,
+        vec![0, 1],
+    );
+
+    let source_state_handler = SourceStateTableHandler::from_table_catalog(
+        &default_source_internal_table(0x2333),
+        MemoryStateStore::new(),
+    )
+    .await;
+
+    let cdc_backfill = CdcBackfillExecutor::new(
+        ActorContext::create(0x1a),
+        external_table,
+        Box::new(mock_offset_executor),
+        vec![0, 1, 2],
+        None,
+        schema.clone(),
+        vec![0],
+        Arc::new(StreamingMetrics::unused()),
+        source_state_handler,
+        4, // 4 rows in a snapshot chunk
+    );
+
+    // Create a `MaterializeExecutor` to write the changes to storage.
+    let mut materialize = MaterializeExecutor::for_test(
+        Box::new(cdc_backfill),
+        memory_state_store.clone(),
+        table_id,
+        vec![ColumnOrder::new(0, OrderType::ascending())],
+        column_ids.clone(),
+        4,
+        Arc::new(AtomicU64::new(0)),
+        ConflictBehavior::Overwrite,
+    )
+    .await
+    .boxed()
+    .execute();
+
+    // push upstream chunks
+    let stream_chunk1 = StreamChunk::from_pretty(
+        " I F
+            + 1 10.01
+            + 2 2.02
+            + 3 3.03 // binlog pos=2
+            + 4 4.04
+            + 5 5.05 // binlog pos=4
+            + 6 6.06",
+    );
+    let stream_chunk2 = StreamChunk::from_pretty(
+        " I F
+            + 6 10.08
+            + 199 40.5
+            + 978 72.6
+            + 134 41.7",
+    );
+
+    // The first barrier
+    let curr_epoch = 11;
+    tx.push_barrier(curr_epoch, false);
+
+    // assert barrier is forwarded to mview
+    assert!(matches!(
+        materialize.next().await.unwrap()?,
+        Message::Barrier(Barrier {
+            epoch,
+            mutation: None,
+            ..
+        }) if epoch.curr == curr_epoch
+    ));
+
+    // start the stream pipeline src -> backfill -> mview
+    let mview_handle = tokio::spawn(consume_message_stream(materialize));
+
+    // ingest data and barrier
+    let interval = Duration::from_millis(10);
+    tx.push_chunk(stream_chunk1);
+
+    tokio::time::sleep(interval).await;
+    tx.push_barrier(curr_epoch + 1, false);
+
+    tx.push_chunk(stream_chunk2);
+
+    tokio::time::sleep(interval).await;
+    tx.push_barrier(curr_epoch + 2, false);
+
+    tokio::time::sleep(interval).await;
+    tx.push_barrier(curr_epoch + 3, true);
+
+    // scan the final result of the mv table
+    let column_descs = vec![
+        ColumnDesc::unnamed(ColumnId::from(0), schema[0].data_type.clone()),
+        ColumnDesc::unnamed(ColumnId::from(1), schema[1].data_type.clone()),
+    ];
+    let value_indices = (0..column_descs.len()).collect_vec();
+    // Since we have not polled `Materialize`, we cannot scan anything from this table
+    let table = StorageTable::for_test(
+        memory_state_store.clone(),
+        table_id,
+        column_descs.clone(),
+        vec![OrderType::ascending()],
+        vec![0],
+        value_indices,
+    );
+
+    let scan = Box::new(RowSeqScanExecutor::new(
+        table.clone(),
+        vec![ScanRange::full()],
+        true,
+        to_committed_batch_query_epoch(u64::MAX),
+        1024,
+        "RowSeqExecutor2".to_string(),
+        None,
+    ));
+    let mut stream = scan.execute();
+    while let Some(message) = stream.next().await {
+        println!("[scan] chunk: {:#?}", message.unwrap());
+    }
+
+    mview_handle.await.unwrap()?;
+
+    Ok(())
+}
+
+async fn consume_message_stream(mut stream: BoxedMessageStream) -> StreamResult<()> {
+    while let Some(message) = stream.next().await {
+        let message = message?;
+        match message {
+            Message::Watermark(_) => {
+                break;
+            }
+            Message::Chunk(c) => {
+                println!("[mv] chunk: {:#?}", c);
+            }
+            Message::Barrier(b) => {
+                if let Some(m) = b.mutation && matches!(*m, Mutation::Stop(_)) {
+                    println!("encounter stop barrier");
+                    break
+                }
+            }
+        }
+    }
+    Ok(())
+}
diff --git a/src/config/backfill.toml b/src/config/ci-backfill.toml
similarity index 100%
rename from src/config/backfill.toml
rename to src/config/ci-backfill.toml
diff --git a/src/config/ci-mem.toml b/src/config/ci-mem.toml
new file mode 100644
index 0000000000000..1cabc16540bdf
--- /dev/null
+++ b/src/config/ci-mem.toml
@@ -0,0 +1,20 @@
+[meta]
+disable_recovery = true
+max_heartbeat_interval_secs = 600
+
+[streaming]
+in_flight_barrier_nums = 10
+
+[streaming.developer]
+stream_exchange_concurrent_barriers = 10
+
+[batch]
+enable_barrier_read = true
+
+[storage]
+imm_merge_threshold = 2
+
+[system]
+barrier_interval_ms = 250
+checkpoint_frequency = 5
+max_concurrent_creating_streaming_jobs = 0
diff --git a/src/config/ci-sim.toml b/src/config/ci-sim.toml
index 0890807018acb..94f28684418c8 100644
--- a/src/config/ci-sim.toml
+++ b/src/config/ci-sim.toml
@@ -1,9 +1,10 @@
 [server]
 telemetry_enabled = false
+metrics_level = "Disabled"
 
 [system]
 telemetry_enabled = false
 max_concurrent_creating_streaming_jobs = 0
 
 [meta]
-meta_leader_lease_secs = 10
\ No newline at end of file
+meta_leader_lease_secs = 10
diff --git a/src/config/example.toml b/src/config/example.toml
index 9309f76d335ae..4789f03901649 100644
--- a/src/config/example.toml
+++ b/src/config/example.toml
@@ -3,7 +3,7 @@
 [server]
 heartbeat_interval_ms = 1000
 connection_pool_size = 16
-metrics_level = 0
+metrics_level = "Info"
 telemetry_enabled = true
 
 [server.auto_dump_heap_profile]
@@ -56,7 +56,7 @@ level0_max_compact_file_number = 96
 tombstone_reclaim_ratio = 40
 
 [batch]
-enable_barrier_read = true
+enable_barrier_read = false
 
 [batch.developer]
 batch_connector_message_buffer_size = 16
@@ -92,7 +92,7 @@ sstable_id_remote_fetch_number = 10
 min_sst_size_for_streaming_upload = 33554432
 max_sub_compaction = 4
 max_concurrent_compaction_task_number = 16
-max_preload_wait_time_mill = 10
+max_preload_wait_time_mill = 0
 object_store_streaming_read_timeout_ms = 600000
 object_store_streaming_upload_timeout_ms = 600000
 object_store_upload_timeout_ms = 3600000
@@ -121,7 +121,6 @@ lfu_tiny_lru_capacity_ratio = 0.01
 rated_random_rate_mb = 0
 flush_rate_limit_mb = 0
 reclaim_rate_limit_mb = 0
-refill_levels = []
 
 [storage.meta_file_cache]
 dir = ""
@@ -137,7 +136,11 @@ lfu_tiny_lru_capacity_ratio = 0.01
 rated_random_rate_mb = 0
 flush_rate_limit_mb = 0
 reclaim_rate_limit_mb = 0
-refill_levels = []
+
+[storage.cache_refill]
+data_refill_levels = []
+timeout_ms = 6000
+concurrency = 100
 
 [system]
 barrier_interval_ms = 1000
@@ -150,3 +153,4 @@ backup_storage_url = "memory"
 backup_storage_directory = "backup"
 telemetry_enabled = true
 max_concurrent_creating_streaming_jobs = 1
+pause_on_next_bootstrap = false
diff --git a/src/config/full-iceberg-bench.toml b/src/config/full-iceberg-bench.toml
index cfc3417194cb6..6b0ea54eef72b 100644
--- a/src/config/full-iceberg-bench.toml
+++ b/src/config/full-iceberg-bench.toml
@@ -3,7 +3,7 @@
 [server]
 heartbeat_interval_ms = 1000
 connection_pool_size = 16
-metrics_level = 0
+metrics_level = "Info"
 telemetry_enabled = true
 
 [server.auto_dump_heap_profile]
diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml
index 1dbfe900fab4f..9c6f0b5c8fc84 100644
--- a/src/connector/Cargo.toml
+++ b/src/connector/Cargo.toml
@@ -21,8 +21,9 @@ apache-avro = { git = "https://github.com/risingwavelabs/avro", branch = "idx0de
     "bzip",
     "xz",
 ] }
-arrow-array = "45"
-arrow-schema = "45"
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
+async-nats = "0.31.0"
 async-trait = "0.1"
 auto_enums = { version = "0.8", features = ["futures03"] }
 auto_impl = "1"
@@ -47,18 +48,24 @@ enum-as-inner = "0.6"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
 glob = "0.3"
-google-cloud-pubsub = "0.19"
+google-cloud-pubsub = "0.20"
+hyper = "0.14"
+hyper-tls = "0.5"
 icelake = { workspace = true }
 itertools = "0.11"
+jsonschema-transpiler = "1.10.0"
 maplit = "1.0.2"
 moka = { version = "0.11", features = ["future"] }
+mysql_async = { version = "0.31", default-features = false, features = ["default"] }
+mysql_common = { version = "0.29.2", default-features = false, features = ["chrono"] }
 nexmark = { version = "0.2", features = ["serde"] }
 num-bigint = "0.4"
 opendal = "0.39"
 parking_lot = "0.12"
+paste = "1"
 prometheus = { version = "0.13", features = ["process"] }
-prost = { version = "0.11.9", features = ["no-recursion-limit"] }
-prost-reflect = "0.11.4"
+prost = { version = "0.11", features = ["no-recursion-limit"] }
+prost-reflect = "0.11"
 protobuf-native = "0.2.1"
 pulsar = { version = "6.0", default-features = false, features = [
     "tokio-runtime",
@@ -107,9 +114,13 @@ workspace-hack = { path = "../workspace-hack" }
 
 [dev-dependencies]
 criterion = { workspace = true, features = ["async_tokio", "async"] }
+prost-types = "0.11"
 rand = "0.8"
 tempfile = "3"
 
+[build-dependencies]
+prost-build = "0.11"
+
 [[bench]]
 name = "parser"
 harness = false
diff --git a/src/connector/benches/parser.rs b/src/connector/benches/parser.rs
index 35851d0ff5f2b..2c3b666b91d62 100644
--- a/src/connector/benches/parser.rs
+++ b/src/connector/benches/parser.rs
@@ -16,7 +16,10 @@ use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criteri
 use maplit::hashmap;
 use rand::Rng;
 use risingwave_common::types::DataType;
-use risingwave_connector::parser::{JsonParser, SourceStreamChunkBuilder};
+use risingwave_connector::parser::{
+    EncodingProperties, JsonParser, JsonProperties, ProtocolProperties, SourceStreamChunkBuilder,
+    SpecificParserConfig,
+};
 use risingwave_connector::source::SourceColumnDesc;
 use serde_json::json;
 use tokio::runtime::Runtime;
@@ -70,7 +73,14 @@ fn create_parser(
         SourceColumnDesc::simple("charlie", DataType::Int64, 2.into()),
         SourceColumnDesc::simple("delta", DataType::Int64, 3.into()),
     ];
-    let parser = JsonParser::new(desc.clone(), Default::default()).unwrap();
+    let props = SpecificParserConfig {
+        key_encoding_config: None,
+        encoding_config: EncodingProperties::Json(JsonProperties {
+            use_schema_registry: false,
+        }),
+        protocol_config: ProtocolProperties::Plain,
+    };
+    let parser = JsonParser::new(props, desc.clone(), Default::default()).unwrap();
     let input = gen_input(mode, chunk_size, chunk_num);
     (parser, desc, input)
 }
diff --git a/src/connector/build.rs b/src/connector/build.rs
new file mode 100644
index 0000000000000..6439fcf00f932
--- /dev/null
+++ b/src/connector/build.rs
@@ -0,0 +1,29 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+fn main() {
+    let proto_dir = "./src/test_data/proto_recursive";
+
+    println!("cargo:rerun-if-changed={}", proto_dir);
+
+    let proto_files = vec!["recursive"];
+    let protos: Vec<String> = proto_files
+        .iter()
+        .map(|f| format!("{}/{}.proto", proto_dir, f))
+        .collect();
+    prost_build::Config::new()
+        .out_dir("./src/parser/protobuf")
+        .compile_protos(&protos, &Vec::<String>::new())
+        .unwrap();
+}
diff --git a/src/connector/src/common.rs b/src/connector/src/common.rs
index 35a9a3451e000..5a03fc7bfd9af 100644
--- a/src/connector/src/common.rs
+++ b/src/connector/src/common.rs
@@ -17,15 +17,19 @@ use std::collections::HashMap;
 use std::time::Duration;
 
 use anyhow::Ok;
+use async_nats::jetstream::consumer::DeliverPolicy;
+use async_nats::jetstream::{self};
 use aws_sdk_kinesis::Client as KinesisClient;
 use clickhouse::Client;
 use rdkafka::ClientConfig;
+use risingwave_common::error::anyhow_error;
 use serde_derive::{Deserialize, Serialize};
 use serde_with::json::JsonString;
 use serde_with::{serde_as, DisplayFromStr};
 
 use crate::aws_auth::AwsAuthProps;
 use crate::deserialize_duration_from_string;
+use crate::sink::SinkError;
 
 // The file describes the common abstractions for each connector and can be used in both source and
 // sink.
@@ -307,12 +311,22 @@ pub struct ClickHouseCommon {
     pub table: String,
 }
 
+const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(5);
+
 impl ClickHouseCommon {
     pub(crate) fn build_client(&self) -> anyhow::Result<Client> {
-        let client = Client::default()
+        use hyper_tls::HttpsConnector;
+
+        let https = HttpsConnector::new();
+        let client = hyper::Client::builder()
+            .pool_idle_timeout(POOL_IDLE_TIMEOUT)
+            .build::<_, hyper::Body>(https);
+
+        let client = Client::with_http_client(client)
             .with_url(&self.url)
             .with_user(&self.user)
-            .with_password(&self.password);
+            .with_password(&self.password)
+            .with_database(&self.database);
         Ok(client)
     }
 }
@@ -324,3 +338,118 @@ pub struct UpsertMessage<'a> {
     #[serde(borrow)]
     pub record: Cow<'a, [u8]>,
 }
+
+#[serde_as]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct NatsCommon {
+    #[serde(rename = "nats.server_url")]
+    pub server_url: String,
+    #[serde(rename = "nats.subject")]
+    pub subject: String,
+    #[serde(rename = "nats.user")]
+    pub user: Option<String>,
+    #[serde(rename = "nats.password")]
+    pub password: Option<String>,
+    #[serde(rename = "nats.max_bytes")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub max_bytes: Option<i64>,
+    #[serde(rename = "nats.max_messages")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub max_messages: Option<i64>,
+    #[serde(rename = "nats.max_messages_per_subject")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub max_messages_per_subject: Option<i64>,
+    #[serde(rename = "nats.max_consumers")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub max_consumers: Option<i32>,
+    #[serde(rename = "nats.max_message_size")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub max_message_size: Option<i32>,
+}
+
+impl NatsCommon {
+    pub(crate) async fn build_client(&self) -> anyhow::Result<async_nats::Client> {
+        let mut connect_options = async_nats::ConnectOptions::new();
+        if let (Some(v_user), Some(v_password)) = (self.user.as_ref(), self.password.as_ref()) {
+            connect_options = connect_options.user_and_password(v_user.into(), v_password.into());
+        }
+        let servers = self.server_url.split(',').collect::<Vec<&str>>();
+        let client = connect_options
+            .connect(
+                servers
+                    .iter()
+                    .map(|url| url.parse())
+                    .collect::<Result<Vec<async_nats::ServerAddr>, _>>()?,
+            )
+            .await
+            .map_err(|e| SinkError::Nats(anyhow_error!("build nats client error: {:?}", e)))?;
+        Ok(client)
+    }
+
+    pub(crate) async fn build_context(&self) -> anyhow::Result<jetstream::Context> {
+        let client = self.build_client().await?;
+        let jetstream = async_nats::jetstream::new(client);
+        Ok(jetstream)
+    }
+
+    pub(crate) async fn build_consumer(
+        &self,
+        split_id: i32,
+        start_sequence: Option<u64>,
+    ) -> anyhow::Result<
+        async_nats::jetstream::consumer::Consumer<async_nats::jetstream::consumer::pull::Config>,
+    > {
+        let context = self.build_context().await?;
+        let stream = self.build_or_get_stream(context.clone()).await?;
+        let name = format!("risingwave-consumer-{}-{}", self.subject, split_id);
+        let mut config = jetstream::consumer::pull::Config {
+            ack_policy: jetstream::consumer::AckPolicy::None,
+            ..Default::default()
+        };
+        match start_sequence {
+            Some(v) => {
+                let consumer = stream
+                    .get_or_create_consumer(&name, {
+                        config.deliver_policy = DeliverPolicy::ByStartSequence {
+                            start_sequence: v + 1,
+                        };
+                        config
+                    })
+                    .await?;
+                Ok(consumer)
+            }
+            None => {
+                let consumer = stream.get_or_create_consumer(&name, config).await?;
+                Ok(consumer)
+            }
+        }
+    }
+
+    pub(crate) async fn build_or_get_stream(
+        &self,
+        jetstream: jetstream::Context,
+    ) -> anyhow::Result<jetstream::stream::Stream> {
+        let mut config = jetstream::stream::Config {
+            // the subject default use name value
+            name: self.subject.clone(),
+            ..Default::default()
+        };
+        if let Some(v) = self.max_bytes {
+            config.max_bytes = v;
+        }
+        if let Some(v) = self.max_messages {
+            config.max_messages = v;
+        }
+        if let Some(v) = self.max_messages_per_subject {
+            config.max_messages_per_subject = v;
+        }
+        if let Some(v) = self.max_consumers {
+            config.max_consumers = v;
+        }
+        if let Some(v) = self.max_message_size {
+            config.max_message_size = v;
+        }
+        let stream = jetstream.get_or_create_stream(config).await?;
+        Ok(stream)
+    }
+}
diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs
index 155f0c248d012..73cd27d5801e5 100644
--- a/src/connector/src/error.rs
+++ b/src/connector/src/error.rs
@@ -26,6 +26,15 @@ pub enum ConnectorError {
     #[error("Kafka error: {0}")]
     Kafka(#[from] rdkafka::error::KafkaError),
 
+    #[error("Config error: {0}")]
+    Config(anyhow::Error),
+
+    #[error("Connection error: {0}")]
+    Connection(anyhow::Error),
+
+    #[error("MySQL error: {0}")]
+    MySql(#[from] mysql_async::Error),
+
     #[error(transparent)]
     Internal(#[from] anyhow::Error),
 }
diff --git a/src/connector/src/lib.rs b/src/connector/src/lib.rs
index d9fbad7b9bfb5..75a895a5f80cd 100644
--- a/src/connector/src/lib.rs
+++ b/src/connector/src/lib.rs
@@ -29,6 +29,7 @@
 #![feature(return_position_impl_trait_in_trait)]
 #![feature(async_fn_in_trait)]
 #![feature(associated_type_defaults)]
+#![feature(impl_trait_in_assoc_type)]
 
 use std::time::Duration;
 
diff --git a/src/connector/src/macros.rs b/src/connector/src/macros.rs
index 70fb6130b8717..a30ad58ce07ce 100644
--- a/src/connector/src/macros.rs
+++ b/src/connector/src/macros.rs
@@ -37,7 +37,7 @@ macro_rules! impl_split_enumerator {
                                 .map(SplitImpl::$variant_name)
                                 .collect_vec()
                         })
-                        .map_err(|e| ErrorCode::ConnectorError(e.into()).into()),
+                        .map_err(|e| risingwave_common::error::ErrorCode::ConnectorError(e.into()).into()),
                     )*
                 }
              }
@@ -55,6 +55,25 @@ macro_rules! impl_split {
                 }
             }
         }
+        $(
+            impl TryFrom<SplitImpl> for $split {
+                type Error = anyhow::Error;
+
+                fn try_from(split: SplitImpl) -> std::result::Result<Self, Self::Error> {
+                    match split {
+                        SplitImpl::$variant_name(inner) => Ok(inner),
+                        other => Err(anyhow::anyhow!("expect {} but get {:?}", stringify!($split), other))
+                    }
+                }
+            }
+
+            impl From<$split> for SplitImpl {
+                fn from(split: $split) -> SplitImpl {
+                    SplitImpl::$variant_name(split)
+                }
+            }
+
+        )*
 
         impl TryFrom<&ConnectorSplit> for SplitImpl {
             type Error = anyhow::Error;
@@ -161,7 +180,8 @@ macro_rules! impl_connector_properties {
             pub fn extract(mut props: HashMap<String, String>) -> Result<Self> {
                 const UPSTREAM_SOURCE_KEY: &str = "connector";
                 let connector = props.remove(UPSTREAM_SOURCE_KEY).ok_or_else(|| anyhow!("Must specify 'connector' in WITH clause"))?;
-                if connector.ends_with("cdc") {
+                use $crate::source::cdc::CDC_CONNECTOR_NAME_SUFFIX;
+                if connector.ends_with(CDC_CONNECTOR_NAME_SUFFIX) {
                     ConnectorProperties::new_cdc_properties(&connector, props)
                 } else {
                     let json_value = serde_json::to_value(props).map_err(|e| anyhow!(e))?;
@@ -182,55 +202,87 @@ macro_rules! impl_connector_properties {
 }
 
 #[macro_export]
-macro_rules! impl_common_split_reader_logic {
-    ($reader:ty, $props:ty) => {
-        impl $reader {
-            #[try_stream(boxed, ok = $crate::source::StreamChunkWithState, error = risingwave_common::error::RwError)]
-            pub(crate) async fn into_chunk_stream(self) {
-                let parser_config = self.parser_config.clone();
-                let actor_id = self.source_ctx.source_info.actor_id.to_string();
-                let source_id = self.source_ctx.source_info.source_id.to_string();
-                let metrics = self.source_ctx.metrics.clone();
-                let source_ctx = self.source_ctx.clone();
-
-                let data_stream = self.into_data_stream();
-
-                let data_stream = data_stream
-                .inspect_ok(move |data_batch| {
-                    let mut by_split_id = std::collections::HashMap::new();
-
-                    for msg in data_batch {
-                        by_split_id
-                            .entry(msg.split_id.as_ref())
-                            .or_insert_with(Vec::new)
-                            .push(msg);
+macro_rules! impl_cdc_source_type {
+    ($({$source_type:ident, $name:expr }),*) => {
+        $(
+            paste!{
+                #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+                pub struct $source_type;
+                impl CdcSourceTypeTrait for $source_type {
+                    const CDC_CONNECTOR_NAME: &'static str = concat!($name, "-cdc");
+                    fn source_type() -> CdcSourceType {
+                        CdcSourceType::$source_type
                     }
+                }
 
-                    for (split_id, msgs) in by_split_id {
-                        metrics
-                            .partition_input_count
-                            .with_label_values(&[&actor_id, &source_id, split_id])
-                            .inc_by(msgs.len() as u64);
-
-                        let sum_bytes = msgs
-                            .iter()
-                            .flat_map(|msg| msg.payload.as_ref().map(|p| p.len() as u64))
-                            .sum();
-
-                        metrics
-                            .partition_input_bytes
-                            .with_label_values(&[&actor_id, &source_id, &split_id])
-                            .inc_by(sum_bytes);
-                    }
-                }).boxed();
+                pub type [< $source_type DebeziumSplitEnumerator >] = DebeziumSplitEnumerator<$source_type>;
+            }
+        )*
+
+        pub enum CdcSourceType {
+            $(
+                $source_type,
+            )*
+        }
+
+        impl From<PbSourceType> for CdcSourceType {
+            fn from(value: PbSourceType) -> Self {
+                match value {
+                    PbSourceType::Unspecified => unreachable!(),
+                    $(
+                        PbSourceType::$source_type => CdcSourceType::$source_type,
+                    )*
+                }
+            }
+        }
 
-                let parser =
-                    $crate::parser::ByteStreamSourceParserImpl::create(parser_config, source_ctx).await?;
-                #[for_await]
-                for msg_batch in parser.into_stream(data_stream) {
-                    yield msg_batch?;
+        impl From<CdcSourceType> for PbSourceType {
+            fn from(this: CdcSourceType) -> PbSourceType {
+                match this {
+                    $(
+                        CdcSourceType::$source_type => PbSourceType::$source_type,
+                    )*
+                }
+            }
+        }
+
+        impl ConnectorProperties {
+            pub(crate) fn new_cdc_properties(
+                connector_name: &str,
+                properties: HashMap<String, String>,
+            ) -> std::result::Result<Self, anyhow::Error> {
+                match connector_name {
+                    $(
+                        $source_type::CDC_CONNECTOR_NAME => paste! {
+                            Ok(Self::[< $source_type Cdc >](Box::new(CdcProperties::<$source_type> {
+                                props: properties,
+                                ..Default::default()
+                            })))
+                        },
+                    )*
+                    _ => Err(anyhow::anyhow!("unexpected cdc connector '{}'", connector_name,)),
+                }
+            }
+
+            pub fn init_cdc_properties(&mut self, table_schema: PbTableSchema) {
+                match self {
+                    $(
+                         paste! {ConnectorProperties:: [< $source_type Cdc >](c)} => {
+                            c.table_schema = table_schema;
+                         }
+                    )*
+                    _ => {}
+                }
+            }
+
+            pub fn is_cdc_connector(&self) -> bool {
+                match self {
+                    $(
+                         paste! {ConnectorProperties:: [< $source_type Cdc >](_)} => true,
+                    )*
+                    _ => false,
                 }
             }
         }
-    };
+    }
 }
diff --git a/src/connector/src/parser/avro/parser.rs b/src/connector/src/parser/avro/parser.rs
index d6bfddb2dbdf1..0ce94cc1158eb 100644
--- a/src/connector/src/parser/avro/parser.rs
+++ b/src/connector/src/parser/avro/parser.rs
@@ -21,13 +21,15 @@ use risingwave_common::error::ErrorCode::{InternalError, ProtocolError};
 use risingwave_common::error::{Result, RwError};
 use risingwave_common::try_match_expand;
 use risingwave_pb::plan_common::ColumnDesc;
-use url::Url;
 
 use super::schema_resolver::*;
 use super::util::avro_schema_to_column_descs;
-use crate::parser::schema_registry::{extract_schema_id, get_subject_by_strategy, Client};
+use crate::parser::schema_registry::{
+    extract_schema_id, get_subject_by_strategy, handle_sr_list, Client,
+};
 use crate::parser::unified::avro::{AvroAccess, AvroParseOptions};
 use crate::parser::unified::AccessImpl;
+use crate::parser::util::{read_schema_from_http, read_schema_from_local, read_schema_from_s3};
 use crate::parser::{AccessBuilder, EncodingProperties, EncodingType};
 
 // Default avro access builder
@@ -114,9 +116,7 @@ impl AvroParserConfig {
         let avro_config = try_match_expand!(encoding_properties, EncodingProperties::Avro)?;
         let schema_location = &avro_config.row_schema_location;
         let enable_upsert = avro_config.enable_upsert;
-        let url = Url::parse(schema_location).map_err(|e| {
-            InternalError(format!("failed to parse url ({}): {}", schema_location, e))
-        })?;
+        let url = handle_sr_list(schema_location.as_str())?;
         if avro_config.use_schema_registry {
             let client = Client::new(url, &avro_config.client_config)?;
             let resolver = ConfluentSchemaResolver::new(client);
@@ -155,12 +155,13 @@ impl AvroParserConfig {
                     "avro upsert without schema registry is not supported".to_string(),
                 )));
             }
+            let url = url.get(0).unwrap();
             let schema_content = match url.scheme() {
                 "file" => read_schema_from_local(url.path()),
                 "s3" => {
-                    read_schema_from_s3(&url, avro_config.aws_auth_props.as_ref().unwrap()).await
+                    read_schema_from_s3(url, avro_config.aws_auth_props.as_ref().unwrap()).await
                 }
-                "https" | "http" => read_schema_from_http(&url).await,
+                "https" | "http" => read_schema_from_http(url).await,
                 scheme => Err(RwError::from(ProtocolError(format!(
                     "path scheme {} is not supported",
                     scheme
diff --git a/src/connector/src/parser/avro/schema_resolver.rs b/src/connector/src/parser/avro/schema_resolver.rs
index 4b2567e200848..a020109d0c79c 100644
--- a/src/connector/src/parser/avro/schema_resolver.rs
+++ b/src/connector/src/parser/avro/schema_resolver.rs
@@ -12,71 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
 use std::sync::Arc;
 
 use apache_avro::Schema;
 use moka::future::Cache;
-use risingwave_common::error::ErrorCode::{InternalError, InvalidConfigValue, ProtocolError};
+use risingwave_common::error::ErrorCode::ProtocolError;
 use risingwave_common::error::{Result, RwError};
-use url::Url;
 
-use crate::aws_auth::AwsAuthProps;
-use crate::aws_utils::{default_conn_config, s3_client};
 use crate::parser::schema_registry::{Client, ConfluentSchema};
-use crate::parser::util::download_from_http;
-
-const AVRO_SCHEMA_LOCATION_S3_REGION: &str = "region";
-
-/// Read schema from s3 bucket.
-/// S3 file location format: <s3://bucket_name/file_name>
-pub(super) async fn read_schema_from_s3(url: &Url, config: &AwsAuthProps) -> Result<String> {
-    let bucket = url
-        .domain()
-        .ok_or_else(|| RwError::from(InternalError(format!("Illegal Avro schema path {}", url))))?;
-    if config.region.is_none() {
-        return Err(RwError::from(InvalidConfigValue {
-            config_entry: AVRO_SCHEMA_LOCATION_S3_REGION.to_string(),
-            config_value: "NONE".to_string(),
-        }));
-    }
-    let key = url.path().replace('/', "");
-    let sdk_config = config.build_config().await?;
-    let s3_client = s3_client(&sdk_config, Some(default_conn_config()));
-    let response = s3_client
-        .get_object()
-        .bucket(bucket.to_string())
-        .key(key)
-        .send()
-        .await
-        .map_err(|e| RwError::from(InternalError(e.to_string())))?;
-    let body_bytes = response.body.collect().await.map_err(|e| {
-        RwError::from(InternalError(format!(
-            "Read Avro schema file from s3 {}",
-            e
-        )))
-    })?;
-    let schema_bytes = body_bytes.into_bytes().to_vec();
-    String::from_utf8(schema_bytes)
-        .map_err(|e| RwError::from(InternalError(format!("Avro schema not valid utf8 {}", e))))
-}
-
-/// Read avro schema file from local file.For on-premise or testing.
-pub(super) fn read_schema_from_local(path: impl AsRef<Path>) -> Result<String> {
-    std::fs::read_to_string(path.as_ref()).map_err(|e| e.into())
-}
-
-/// Read avro schema file from local file.For common usage.
-pub(super) async fn read_schema_from_http(location: &Url) -> Result<String> {
-    let schema_bytes = download_from_http(location).await?;
-
-    String::from_utf8(schema_bytes.into()).map_err(|e| {
-        RwError::from(InternalError(format!(
-            "read schema string from https failed {}",
-            e
-        )))
-    })
-}
 
 #[derive(Debug)]
 pub struct ConfluentSchemaResolver {
@@ -104,13 +47,19 @@ impl ConfluentSchemaResolver {
     }
 
     pub async fn get_by_subject_name(&self, subject_name: &str) -> Result<Arc<Schema>> {
-        let raw_schema = self
-            .confluent_client
-            .get_schema_by_subject(subject_name)
-            .await?;
+        let raw_schema = self.get_raw_schema_by_subject_name(subject_name).await?;
         self.parse_and_cache_schema(raw_schema).await
     }
 
+    pub async fn get_raw_schema_by_subject_name(
+        &self,
+        subject_name: &str,
+    ) -> Result<ConfluentSchema> {
+        self.confluent_client
+            .get_schema_by_subject(subject_name)
+            .await
+    }
+
     // get the writer schema by id
     pub async fn get(&self, schema_id: i32) -> Result<Arc<Schema>> {
         if let Some(schema) = self.writer_schemas.get(&schema_id) {
diff --git a/src/connector/src/parser/canal/simd_json_parser.rs b/src/connector/src/parser/canal/simd_json_parser.rs
index 17740e4541714..8d2d03bf1611c 100644
--- a/src/connector/src/parser/canal/simd_json_parser.rs
+++ b/src/connector/src/parser/canal/simd_json_parser.rs
@@ -21,7 +21,9 @@ use crate::parser::canal::operators::*;
 use crate::parser::unified::json::{JsonAccess, JsonParseOptions};
 use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer;
 use crate::parser::unified::ChangeEventOperation;
-use crate::parser::{ByteStreamSourceParser, SourceStreamChunkRowWriter, WriteGuard};
+use crate::parser::{
+    ByteStreamSourceParser, JsonProperties, SourceStreamChunkRowWriter, WriteGuard,
+};
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
 
 const DATA: &str = "data";
@@ -32,13 +34,19 @@ const IS_DDL: &str = "isDdl";
 pub struct CanalJsonParser {
     pub(crate) rw_columns: Vec<SourceColumnDesc>,
     source_ctx: SourceContextRef,
+    payload_start_idx: usize,
 }
 
 impl CanalJsonParser {
-    pub fn new(rw_columns: Vec<SourceColumnDesc>, source_ctx: SourceContextRef) -> Result<Self> {
+    pub fn new(
+        rw_columns: Vec<SourceColumnDesc>,
+        source_ctx: SourceContextRef,
+        config: &JsonProperties,
+    ) -> Result<Self> {
         Ok(Self {
             rw_columns,
             source_ctx,
+            payload_start_idx: if config.use_schema_registry { 5 } else { 0 },
         })
     }
 
@@ -48,8 +56,9 @@ impl CanalJsonParser {
         mut payload: Vec<u8>,
         mut writer: SourceStreamChunkRowWriter<'_>,
     ) -> Result<WriteGuard> {
-        let mut event: BorrowedValue<'_> = simd_json::to_borrowed_value(&mut payload)
-            .map_err(|e| RwError::from(ProtocolError(e.to_string())))?;
+        let mut event: BorrowedValue<'_> =
+            simd_json::to_borrowed_value(&mut payload[self.payload_start_idx..])
+                .map_err(|e| RwError::from(ProtocolError(e.to_string())))?;
 
         let is_ddl = event.get(IS_DDL).and_then(|v| v.as_bool()).ok_or_else(|| {
             RwError::from(ProtocolError(
@@ -151,7 +160,12 @@ mod tests {
             SourceColumnDesc::simple("binary", DataType::Bytea, 6.into()),
             SourceColumnDesc::simple("json", DataType::Jsonb, 7.into()),
         ];
-        let parser = CanalJsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = CanalJsonParser::new(
+            descs.clone(),
+            Default::default(),
+            &JsonProperties::default(),
+        )
+        .unwrap();
 
         let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 1);
 
@@ -217,7 +231,12 @@ mod tests {
             SourceColumnDesc::simple("win_rate", DataType::Float64, 5.into()),
         ];
 
-        let parser = CanalJsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = CanalJsonParser::new(
+            descs.clone(),
+            Default::default(),
+            &JsonProperties::default(),
+        )
+        .unwrap();
 
         let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 2);
 
@@ -266,7 +285,12 @@ mod tests {
             SourceColumnDesc::simple("v2", DataType::Int32, 1.into()),
         ];
 
-        let parser = CanalJsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = CanalJsonParser::new(
+            descs.clone(),
+            Default::default(),
+            &JsonProperties::default(),
+        )
+        .unwrap();
 
         let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 2);
 
diff --git a/src/connector/src/parser/debezium/avro_parser.rs b/src/connector/src/parser/debezium/avro_parser.rs
index 2fc3c3c8fcff5..a83b1443b5b1e 100644
--- a/src/connector/src/parser/debezium/avro_parser.rs
+++ b/src/connector/src/parser/debezium/avro_parser.rs
@@ -17,15 +17,14 @@ use std::sync::Arc;
 
 use apache_avro::types::Value;
 use apache_avro::{from_avro_datum, Schema};
-use reqwest::Url;
-use risingwave_common::error::ErrorCode::{InternalError, ProtocolError};
+use risingwave_common::error::ErrorCode::ProtocolError;
 use risingwave_common::error::{Result, RwError};
 use risingwave_common::try_match_expand;
 use risingwave_pb::plan_common::ColumnDesc;
 
 use crate::parser::avro::schema_resolver::ConfluentSchemaResolver;
 use crate::parser::avro::util::avro_schema_to_column_descs;
-use crate::parser::schema_registry::{extract_schema_id, Client};
+use crate::parser::schema_registry::{extract_schema_id, handle_sr_list, Client};
 use crate::parser::unified::avro::{
     avro_extract_field_schema, avro_schema_skip_union, AvroAccess, AvroParseOptions,
 };
@@ -107,9 +106,7 @@ impl DebeziumAvroParserConfig {
         let schema_location = &avro_config.row_schema_location;
         let client_config = &avro_config.client_config;
         let kafka_topic = &avro_config.topic;
-        let url = Url::parse(schema_location).map_err(|e| {
-            InternalError(format!("failed to parse url ({}): {}", schema_location, e))
-        })?;
+        let url = handle_sr_list(schema_location)?;
         let client = Client::new(url, client_config)?;
         let raw_schema = client
             .get_schema_by_subject(format!("{}-key", &kafka_topic).as_str())
diff --git a/src/connector/src/parser/debezium/debezium_parser.rs b/src/connector/src/parser/debezium/debezium_parser.rs
index 62ca2255d8095..8fd0f6f2ba4f6 100644
--- a/src/connector/src/parser/debezium/debezium_parser.rs
+++ b/src/connector/src/parser/debezium/debezium_parser.rs
@@ -80,7 +80,9 @@ impl DebeziumParser {
     pub async fn new_for_test(rw_columns: Vec<SourceColumnDesc>) -> Result<Self> {
         let props = SpecificParserConfig {
             key_encoding_config: None,
-            encoding_config: EncodingProperties::Json(JsonProperties {}),
+            encoding_config: EncodingProperties::Json(JsonProperties {
+                use_schema_registry: false,
+            }),
             protocol_config: ProtocolProperties::Debezium,
         };
         Self::new(props, rw_columns, Default::default()).await
diff --git a/src/connector/src/parser/debezium/simd_json_parser.rs b/src/connector/src/parser/debezium/simd_json_parser.rs
index d69371a4c9292..0bfd69a7bb6fe 100644
--- a/src/connector/src/parser/debezium/simd_json_parser.rs
+++ b/src/connector/src/parser/debezium/simd_json_parser.rs
@@ -90,7 +90,9 @@ mod tests {
     ) -> DebeziumParser {
         let props = SpecificParserConfig {
             key_encoding_config: None,
-            encoding_config: EncodingProperties::Json(JsonProperties {}),
+            encoding_config: EncodingProperties::Json(JsonProperties {
+                use_schema_registry: false,
+            }),
             protocol_config: ProtocolProperties::Debezium,
         };
         DebeziumParser::new(props, rw_columns, source_ctx)
@@ -296,7 +298,7 @@ mod tests {
                 SourceColumnDesc::simple("O_DATE", DataType::Date, ColumnId::from(8)),
                 SourceColumnDesc::simple("O_TIME", DataType::Time, ColumnId::from(9)),
                 SourceColumnDesc::simple("O_DATETIME", DataType::Timestamp, ColumnId::from(10)),
-                SourceColumnDesc::simple("O_TIMESTAMP", DataType::Timestamp, ColumnId::from(11)),
+                SourceColumnDesc::simple("O_TIMESTAMP", DataType::Timestamptz, ColumnId::from(11)),
                 SourceColumnDesc::simple("O_JSON", DataType::Jsonb, ColumnId::from(12)),
             ]
         }
@@ -331,9 +333,9 @@ mod tests {
             assert!(row[10].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
                 "1970-01-01T00:00:00".parse().unwrap()
             )))));
-            assert!(row[11].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
-                "1970-01-01T00:00:01".parse().unwrap()
-            )))));
+            assert!(row[11].eq(&Some(ScalarImpl::Timestamptz(
+                "1970-01-01T00:00:01Z".parse().unwrap()
+            ))));
             assert_json_eq(&row[12], "{\"k1\": \"v1\", \"k2\": 11}");
         }
 
@@ -366,9 +368,9 @@ mod tests {
             assert!(row[10].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
                 "1970-01-01T00:00:00".parse().unwrap()
             )))));
-            assert!(row[11].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
-                "1970-01-01T00:00:01".parse().unwrap()
-            )))));
+            assert!(row[11].eq(&Some(ScalarImpl::Timestamptz(
+                "1970-01-01T00:00:01Z".parse().unwrap()
+            ))));
             assert_json_eq(&row[12], "{\"k1\": \"v1\", \"k2\": 11}");
         }
 
@@ -402,9 +404,9 @@ mod tests {
             assert!(row[10].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
                 "5138-11-16T09:46:39".parse().unwrap()
             )))));
-            assert!(row[11].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
-                "2038-01-09T03:14:07".parse().unwrap()
-            )))));
+            assert!(row[11].eq(&Some(ScalarImpl::Timestamptz(
+                "2038-01-09T03:14:07Z".parse().unwrap()
+            ))));
             assert_json_eq(&row[12], "{\"k1\":\"v1_updated\",\"k2\":33}");
         }
 
@@ -439,9 +441,9 @@ mod tests {
             assert!(row[10].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
                 "5138-11-16T09:46:39".parse().unwrap()
             )))));
-            assert!(row[11].eq(&Some(ScalarImpl::Timestamp(Timestamp::new(
-                "2038-01-09T03:14:07".parse().unwrap()
-            )))));
+            assert!(row[11].eq(&Some(ScalarImpl::Timestamptz(
+                "2038-01-09T03:14:07Z".parse().unwrap()
+            ))));
             assert_json_eq(&row[12], "{\"k1\": \"v1_updated\", \"k2\": 33}");
         }
 
@@ -537,6 +539,7 @@ mod tests {
     // postgres-specific data-type mapping tests
     mod test3_postgres {
         use super::*;
+        use crate::source::SourceColumnType;
 
         // schema for temporal-type test
         fn get_temporal_test_columns() -> Vec<SourceColumnDesc> {
@@ -596,8 +599,7 @@ mod tests {
                     ])),
                     column_id: 7.into(),
                     fields: vec![],
-                    is_row_id: false,
-                    is_meta: false,
+                    column_type: SourceColumnType::Normal,
                     is_pk: false,
                 },
                 SourceColumnDesc::simple("o_enum", DataType::Varchar, ColumnId::from(8)),
diff --git a/src/connector/src/parser/json_parser.rs b/src/connector/src/parser/json_parser.rs
index 283cd11bd95eb..7c59318e960a1 100644
--- a/src/connector/src/parser/json_parser.rs
+++ b/src/connector/src/parser/json_parser.rs
@@ -12,28 +12,43 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::error::ErrorCode::{self, ProtocolError};
-use risingwave_common::error::{Result, RwError};
+use std::collections::HashMap;
 
-use super::unified::json::JsonParseOptions;
-use super::unified::util::apply_row_accessor_on_stream_chunk_writer;
-use super::unified::AccessImpl;
-use super::{AccessBuilder, ByteStreamSourceParser};
-use crate::parser::unified::json::JsonAccess;
-use crate::parser::{SourceStreamChunkRowWriter, WriteGuard};
+use apache_avro::Schema;
+use jst::{convert_avro, Context};
+use risingwave_common::error::ErrorCode::{self, InternalError, ProtocolError};
+use risingwave_common::error::{Result, RwError};
+use risingwave_common::try_match_expand;
+use risingwave_pb::plan_common::ColumnDesc;
+
+use super::avro::schema_resolver::ConfluentSchemaResolver;
+use super::schema_registry::Client;
+use super::util::{get_kafka_topic, read_schema_from_http, read_schema_from_local};
+use super::{EncodingProperties, SchemaRegistryAuth, SpecificParserConfig};
+use crate::parser::avro::util::avro_schema_to_column_descs;
+use crate::parser::schema_registry::handle_sr_list;
+use crate::parser::unified::json::{JsonAccess, JsonParseOptions};
+use crate::parser::unified::util::apply_row_accessor_on_stream_chunk_writer;
+use crate::parser::unified::AccessImpl;
+use crate::parser::{
+    AccessBuilder, ByteStreamSourceParser, SourceStreamChunkRowWriter, WriteGuard,
+};
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
 
 #[derive(Debug)]
 pub struct JsonAccessBuilder {
     value: Option<Vec<u8>>,
+    payload_start_idx: usize,
 }
 
 impl AccessBuilder for JsonAccessBuilder {
     #[allow(clippy::unused_async)]
     async fn generate_accessor(&mut self, payload: Vec<u8>) -> Result<AccessImpl<'_, '_>> {
         self.value = Some(payload);
-        let value = simd_json::to_borrowed_value(self.value.as_mut().unwrap())
-            .map_err(|e| RwError::from(ProtocolError(e.to_string())))?;
+        let value = simd_json::to_borrowed_value(
+            &mut self.value.as_mut().unwrap()[self.payload_start_idx..],
+        )
+        .map_err(|e| RwError::from(ProtocolError(e.to_string())))?;
         Ok(AccessImpl::Json(JsonAccess::new_with_options(
             value,
             // Debezium and Canal have their special json access builder and will not
@@ -44,8 +59,11 @@ impl AccessBuilder for JsonAccessBuilder {
 }
 
 impl JsonAccessBuilder {
-    pub fn new() -> Result<Self> {
-        Ok(Self { value: None })
+    pub fn new(use_schema_registry: bool) -> Result<Self> {
+        Ok(Self {
+            value: None,
+            payload_start_idx: if use_schema_registry { 5 } else { 0 },
+        })
     }
 }
 
@@ -54,13 +72,26 @@ impl JsonAccessBuilder {
 pub struct JsonParser {
     rw_columns: Vec<SourceColumnDesc>,
     source_ctx: SourceContextRef,
+    // If schema registry is used, the starting index of payload is 5.
+    payload_start_idx: usize,
 }
 
 impl JsonParser {
-    pub fn new(rw_columns: Vec<SourceColumnDesc>, source_ctx: SourceContextRef) -> Result<Self> {
+    pub fn new(
+        props: SpecificParserConfig,
+        rw_columns: Vec<SourceColumnDesc>,
+        source_ctx: SourceContextRef,
+    ) -> Result<Self> {
+        let json_config = try_match_expand!(props.encoding_config, EncodingProperties::Json)?;
+        let payload_start_idx = if json_config.use_schema_registry {
+            5
+        } else {
+            0
+        };
         Ok(Self {
             rw_columns,
             source_ctx,
+            payload_start_idx,
         })
     }
 
@@ -68,6 +99,7 @@ impl JsonParser {
         Ok(Self {
             rw_columns,
             source_ctx: Default::default(),
+            payload_start_idx: 0,
         })
     }
 
@@ -82,8 +114,9 @@ impl JsonParser {
                 "Empty payload with nonempty key for non-upsert".into(),
             )));
         }
-        let value = simd_json::to_borrowed_value(payload.as_mut().unwrap())
-            .map_err(|e| RwError::from(ProtocolError(e.to_string())))?;
+        let value =
+            simd_json::to_borrowed_value(&mut payload.as_mut().unwrap()[self.payload_start_idx..])
+                .map_err(|e| RwError::from(ProtocolError(e.to_string())))?;
         let values = if let simd_json::BorrowedValue::Array(arr) = value {
             arr
         } else {
@@ -113,6 +146,40 @@ impl JsonParser {
     }
 }
 
+pub async fn schema_to_columns(
+    schema_location: &str,
+    use_schema_registry: bool,
+    props: &HashMap<String, String>,
+) -> anyhow::Result<Vec<ColumnDesc>> {
+    let url = handle_sr_list(schema_location)?;
+    let schema_content = if use_schema_registry {
+        let schema_registry_auth = SchemaRegistryAuth::from(props);
+        let client = Client::new(url, &schema_registry_auth)?;
+        let topic = get_kafka_topic(props)?;
+        let resolver = ConfluentSchemaResolver::new(client);
+        resolver
+            .get_raw_schema_by_subject_name(&format!("{}-value", topic))
+            .await?
+            .content
+    } else {
+        let url = url.get(0).unwrap();
+        match url.scheme() {
+            "file" => read_schema_from_local(url.path()),
+            "https" | "http" => read_schema_from_http(url).await,
+            scheme => Err(RwError::from(ProtocolError(format!(
+                "path scheme {} is not supported",
+                scheme
+            )))),
+        }?
+    };
+    let json_schema = serde_json::from_str(&schema_content)?;
+    let context = Context::default();
+    let avro_schema = convert_avro(&json_schema, context).to_string();
+    let schema = Schema::parse_str(&avro_schema)
+        .map_err(|e| RwError::from(InternalError(format!("Avro schema parse error {}", e))))?;
+    avro_schema_to_column_descs(&schema)
+}
+
 impl ByteStreamSourceParser for JsonParser {
     fn columns(&self) -> &[SourceColumnDesc] {
         &self.rw_columns
@@ -145,9 +212,10 @@ mod tests {
     use risingwave_common::test_prelude::StreamChunkTestExt;
     use risingwave_common::types::{DataType, Decimal, ScalarImpl, ToOwnedDatum};
 
+    use super::JsonParser;
     use crate::parser::upsert_parser::UpsertParser;
     use crate::parser::{
-        EncodingProperties, JsonParser, JsonProperties, ProtocolProperties, SourceColumnDesc,
+        EncodingProperties, JsonProperties, ProtocolProperties, SourceColumnDesc,
         SourceStreamChunkBuilder, SpecificParserConfig,
     };
 
@@ -178,7 +246,12 @@ mod tests {
             SourceColumnDesc::simple("decimal", DataType::Decimal, 10.into()),
         ];
 
-        let parser = JsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = JsonParser::new(
+            SpecificParserConfig::DEFAULT_PLAIN_JSON,
+            descs.clone(),
+            Default::default(),
+        )
+        .unwrap();
 
         let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 2);
 
@@ -277,7 +350,12 @@ mod tests {
             SourceColumnDesc::simple("v2", DataType::Int16, 1.into()),
             SourceColumnDesc::simple("v3", DataType::Varchar, 2.into()),
         ];
-        let parser = JsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = JsonParser::new(
+            SpecificParserConfig::DEFAULT_PLAIN_JSON,
+            descs.clone(),
+            Default::default(),
+        )
+        .unwrap();
         let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 3);
 
         // Parse a correct record.
@@ -343,7 +421,12 @@ mod tests {
         .map(SourceColumnDesc::from)
         .collect_vec();
 
-        let parser = JsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = JsonParser::new(
+            SpecificParserConfig::DEFAULT_PLAIN_JSON,
+            descs.clone(),
+            Default::default(),
+        )
+        .unwrap();
         let payload = br#"
         {
             "data": {
@@ -410,7 +493,12 @@ mod tests {
         .map(SourceColumnDesc::from)
         .collect_vec();
 
-        let parser = JsonParser::new(descs.clone(), Default::default()).unwrap();
+        let parser = JsonParser::new(
+            SpecificParserConfig::DEFAULT_PLAIN_JSON,
+            descs.clone(),
+            Default::default(),
+        )
+        .unwrap();
         let payload = br#"
         {
             "struct": "{\"varchar\": \"varchar\", \"boolean\": true}"
@@ -449,7 +537,9 @@ mod tests {
         ];
         let props = SpecificParserConfig {
             key_encoding_config: None,
-            encoding_config: EncodingProperties::Json(JsonProperties {}),
+            encoding_config: EncodingProperties::Json(JsonProperties {
+                use_schema_registry: false,
+            }),
             protocol_config: ProtocolProperties::Upsert,
         };
         let mut parser = UpsertParser::new(props, descs.clone(), Default::default())
diff --git a/src/connector/src/parser/maxwell/simd_json_parser.rs b/src/connector/src/parser/maxwell/simd_json_parser.rs
index 4eccd33bbdc39..c804171c10863 100644
--- a/src/connector/src/parser/maxwell/simd_json_parser.rs
+++ b/src/connector/src/parser/maxwell/simd_json_parser.rs
@@ -35,7 +35,9 @@ mod tests {
 
         let props = SpecificParserConfig {
             key_encoding_config: None,
-            encoding_config: EncodingProperties::Json(JsonProperties {}),
+            encoding_config: EncodingProperties::Json(JsonProperties {
+                use_schema_registry: false,
+            }),
             protocol_config: ProtocolProperties::Maxwell,
         };
         let mut parser = MaxwellParser::new(props, descs.clone(), Default::default())
diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs
index e19df8cd37e2d..ba9f1710a2e55 100644
--- a/src/connector/src/parser/mod.rs
+++ b/src/connector/src/parser/mod.rs
@@ -38,6 +38,7 @@ pub use schema_registry::name_strategy_from_str;
 
 use self::avro::AvroAccessBuilder;
 use self::bytes_parser::BytesAccessBuilder;
+pub use self::mysql::mysql_row_to_datums;
 use self::plain_parser::PlainParser;
 use self::simd_json_parser::DebeziumJsonAccessBuilder;
 use self::unified::AccessImpl;
@@ -58,6 +59,7 @@ mod csv_parser;
 mod debezium;
 mod json_parser;
 mod maxwell;
+mod mysql;
 mod plain_parser;
 mod protobuf;
 mod schema_registry;
@@ -134,7 +136,7 @@ pub struct SourceStreamChunkRowWriter<'a> {
 
 /// `WriteGuard` can't be constructed directly in other mods due to a private field, so it can be
 /// used to ensure that all methods on [`SourceStreamChunkRowWriter`] are called at least once in
-/// the [`SourceParser::parse`] implementation.
+/// the `SourceParser::parse` implementation.
 #[derive(Debug)]
 pub struct WriteGuard(());
 
@@ -169,7 +171,7 @@ impl OpAction for OpActionInsert {
 
     #[inline(always)]
     fn finish(writer: &mut SourceStreamChunkRowWriter<'_>) {
-        writer.op_builder.push(Op::Insert)
+        writer.op_builder.push(Op::Insert);
     }
 }
 
@@ -192,7 +194,7 @@ impl OpAction for OpActionDelete {
 
     #[inline(always)]
     fn finish(writer: &mut SourceStreamChunkRowWriter<'_>) {
-        writer.op_builder.push(Op::Delete)
+        writer.op_builder.push(Op::Delete);
     }
 }
 
@@ -237,10 +239,10 @@ impl SourceStreamChunkRowWriter<'_> {
             .zip_eq(self.builders.iter_mut())
             .enumerate()
             .try_for_each(|(idx, (desc, builder))| -> Result<()> {
-                if desc.is_meta {
+                if desc.is_meta() || desc.is_offset() {
                     return Ok(());
                 }
-                let output = if desc.is_row_id {
+                let output = if desc.is_row_id() {
                     A::DEFAULT_OUTPUT
                 } else {
                     f(desc)?
@@ -277,11 +279,11 @@ impl SourceStreamChunkRowWriter<'_> {
 
     /// For other op like 'insert', 'update', 'delete', we will leave the hollow for the meta column
     /// builder. e.g after insert
-    /// `data_builder` = [1], `meta_column_builder` = [], `op` = [insert]
+    /// `data_builder = [1], meta_column_builder = [], op = [insert]`
     ///
     /// This function is used to fulfill this hollow in `meta_column_builder`.
     /// e.g after fulfill
-    /// `data_builder` = [1], `meta_column_builder` = [1], `op` = [insert]
+    /// `data_builder = [1], meta_column_builder = [1], op = [insert]`
     pub fn fulfill_meta_column(
         &mut self,
         mut f: impl FnMut(&SourceColumnDesc) -> Option<Datum>,
@@ -371,7 +373,7 @@ pub trait ByteStreamSourceParser: Send + Debug + Sized + 'static {
     ///
     /// # Returns
     ///
-    /// A [`BoxSourceWithStateStream`] which is a stream of parsed msgs.
+    /// A [`crate::source::BoxSourceWithStateStream`] which is a stream of parsed msgs.
     fn into_stream(self, data_stream: BoxSourceStream) -> impl SourceWithStateStream {
         into_chunk_stream(self, data_stream)
     }
@@ -435,10 +437,10 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                 continue;
             }
 
-            split_offset_mapping.insert(msg.split_id, msg.offset);
+            let msg_offset = msg.offset;
+            split_offset_mapping.insert(msg.split_id, msg_offset.clone());
 
             let old_op_num = builder.op_num();
-
             match parser
                 .parse_one_with_txn(msg.key, msg.payload, builder.row_writer())
                 .await
@@ -452,26 +454,36 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                         *len += new_op_num - old_op_num;
                     }
 
+                    // fill in meta column for specific source and offset column if needed
                     for _ in old_op_num..new_op_num {
-                        // TODO: support more kinds of SourceMeta
-                        if let SourceMeta::Kafka(kafka_meta) = &msg.meta {
-                            let f = |desc: &SourceColumnDesc| -> Option<risingwave_common::types::Datum> {
-                        if !desc.is_meta {
-                            return None;
-                        }
-                        match desc.name.as_str() {
-                            KAFKA_TIMESTAMP_COLUMN_NAME => Some(kafka_meta.timestamp.map(|ts| {
-                                risingwave_common::cast::i64_to_timestamptz(ts)
-                                    .unwrap()
-                                    .into()
-                            })),
-                            _ => {
-                                unreachable!("kafka will not have this meta column: {}", desc.name)
-                            }
-                        }
-                    };
-                            builder.row_writer().fulfill_meta_column(f)?;
-                        }
+                        let f =
+                            |desc: &SourceColumnDesc| -> Option<risingwave_common::types::Datum> {
+                                if desc.is_meta() && let SourceMeta::Kafka(kafka_meta) = &msg.meta {
+                                    match desc.name.as_str() {
+                                        KAFKA_TIMESTAMP_COLUMN_NAME => {
+                                            Some(kafka_meta.timestamp.map(|ts| {
+                                                risingwave_common::cast::i64_to_timestamptz(ts)
+                                                    .unwrap()
+                                                    .into()
+                                            }))
+                                        }
+                                        _ => {
+                                            unreachable!(
+                                                "kafka will not have this meta column: {}",
+                                                desc.name
+                                            )
+                                        }
+                                    }
+                                } else if desc.is_offset() {
+                                    Some(Some(msg_offset.as_str().into()))
+                                } else {
+                                    // None will be ignored by `fulfill_meta_column`
+                                    None
+                                }
+                            };
+
+                        // fill in meta or offset column if any
+                        builder.row_writer().fulfill_meta_column(f)?;
                     }
                 }
 
@@ -505,8 +517,8 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
 
                 Err(error) => {
                     tracing::warn!(%error, "message parsing failed, skipping");
-                    // This will throw an error for batch
-                    parser.source_ctx().report_user_source_error(error)?;
+                    // Skip for batch
+                    parser.source_ctx().report_user_source_error(error);
                     continue;
                 }
             }
@@ -557,7 +569,9 @@ impl AccessBuilderImpl {
             EncodingProperties::Bytes(_) => {
                 AccessBuilderImpl::Bytes(BytesAccessBuilder::new(config)?)
             }
-            EncodingProperties::Json(_) => AccessBuilderImpl::Json(JsonAccessBuilder::new()?),
+            EncodingProperties::Json(config) => {
+                AccessBuilderImpl::Json(JsonAccessBuilder::new(config.use_schema_registry)?)
+            }
             _ => unreachable!(),
         };
         Ok(accessor)
@@ -615,7 +629,7 @@ impl ByteStreamSourceParserImpl {
         let encode = &parser_config.specific.encoding_config;
         match (protocol, encode) {
             (ProtocolProperties::Plain, EncodingProperties::Json(_)) => {
-                JsonParser::new(rw_columns, source_ctx).map(Self::Json)
+                JsonParser::new(parser_config.specific, rw_columns, source_ctx).map(Self::Json)
             }
             (ProtocolProperties::Plain, EncodingProperties::Csv(config)) => {
                 CsvParser::new(rw_columns, *config, source_ctx).map(Self::Csv)
@@ -623,8 +637,8 @@ impl ByteStreamSourceParserImpl {
             (ProtocolProperties::DebeziumMongo, EncodingProperties::Json(_)) => {
                 DebeziumMongoJsonParser::new(rw_columns, source_ctx).map(Self::DebeziumMongoJson)
             }
-            (ProtocolProperties::Canal, EncodingProperties::Json(_)) => {
-                CanalJsonParser::new(rw_columns, source_ctx).map(Self::CanalJson)
+            (ProtocolProperties::Canal, EncodingProperties::Json(config)) => {
+                CanalJsonParser::new(rw_columns, source_ctx, config).map(Self::CanalJson)
             }
             (ProtocolProperties::Native, _) => unreachable!("Native parser should not be created"),
             (ProtocolProperties::Upsert, _) => {
@@ -676,6 +690,17 @@ pub struct SpecificParserConfig {
     pub protocol_config: ProtocolProperties,
 }
 
+impl SpecificParserConfig {
+    // for test only
+    pub const DEFAULT_PLAIN_JSON: SpecificParserConfig = SpecificParserConfig {
+        key_encoding_config: None,
+        encoding_config: EncodingProperties::Json(JsonProperties {
+            use_schema_registry: false,
+        }),
+        protocol_config: ProtocolProperties::Plain,
+    };
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct SchemaRegistryAuth {
     username: Option<String>,
@@ -728,7 +753,9 @@ pub struct CsvProperties {
 }
 
 #[derive(Debug, Default, Clone)]
-pub struct JsonProperties {}
+pub struct JsonProperties {
+    pub use_schema_registry: bool,
+}
 
 #[derive(Debug, Default, Clone)]
 pub struct BytesProperties {
@@ -885,14 +912,20 @@ impl SpecificParserConfig {
                 })
             }
             (
-                SourceFormat::Debezium
-                | SourceFormat::DebeziumMongo
+                SourceFormat::Plain
+                | SourceFormat::Debezium
                 | SourceFormat::Maxwell
                 | SourceFormat::Canal
-                | SourceFormat::Plain
                 | SourceFormat::Upsert,
                 SourceEncode::Json,
-            ) => EncodingProperties::Json(JsonProperties {}),
+            ) => EncodingProperties::Json(JsonProperties {
+                use_schema_registry: info.use_schema_registry,
+            }),
+            (SourceFormat::DebeziumMongo, SourceEncode::Json) => {
+                EncodingProperties::Json(JsonProperties {
+                    use_schema_registry: false,
+                })
+            }
             (SourceFormat::Plain, SourceEncode::Bytes) => {
                 EncodingProperties::Bytes(BytesProperties { column_name: None })
             }
diff --git a/src/connector/src/parser/mysql.rs b/src/connector/src/parser/mysql.rs
new file mode 100644
index 0000000000000..58be305a69118
--- /dev/null
+++ b/src/connector/src/parser/mysql.rs
@@ -0,0 +1,156 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use chrono::NaiveDate;
+use mysql_async::Row as MysqlRow;
+use risingwave_common::catalog::{Schema, OFFSET_COLUMN_NAME};
+use risingwave_common::types::{
+    DataType, Date, Datum, Decimal, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
+};
+use rust_decimal::Decimal as RustDecimal;
+
+pub fn mysql_row_to_datums(mysql_row: &mut MysqlRow, schema: &Schema) -> Vec<Datum> {
+    let mut datums = vec![];
+    for i in 0..schema.fields.len() {
+        let rw_field = &schema.fields[i];
+        let datum = {
+            match rw_field.data_type {
+                DataType::Boolean => {
+                    let v = mysql_row.take::<bool, _>(i);
+                    v.map(ScalarImpl::from)
+                }
+                DataType::Int16 => {
+                    let v = mysql_row.take::<i16, _>(i);
+                    v.map(ScalarImpl::from)
+                }
+                DataType::Int32 => {
+                    let v = mysql_row.take::<i32, _>(i);
+                    v.map(ScalarImpl::from)
+                }
+                DataType::Int64 => {
+                    let v = mysql_row.take::<i64, _>(i);
+                    v.map(ScalarImpl::from)
+                }
+                DataType::Float32 => {
+                    let v = mysql_row.take::<f32, _>(i);
+                    v.map(ScalarImpl::from)
+                }
+                DataType::Float64 => {
+                    let v = mysql_row.take::<f64, _>(i);
+                    v.map(ScalarImpl::from)
+                }
+                DataType::Decimal => {
+                    let v = mysql_row.take::<RustDecimal, _>(i);
+                    v.map(|v| ScalarImpl::from(Decimal::from(v)))
+                }
+                DataType::Varchar => {
+                    // snapshot data doesn't contain offset, just fill None
+                    if rw_field.name.as_str() == OFFSET_COLUMN_NAME {
+                        None
+                    } else {
+                        let v = mysql_row.take::<String, _>(i);
+                        v.map(ScalarImpl::from)
+                    }
+                }
+                DataType::Date => {
+                    let v = mysql_row.take::<NaiveDate, _>(i);
+                    v.map(|v| ScalarImpl::from(Date::from(v)))
+                }
+                DataType::Time => {
+                    let v = mysql_row.take::<chrono::NaiveTime, _>(i);
+                    v.map(|v| ScalarImpl::from(Time::from(v)))
+                }
+                DataType::Timestamp => {
+                    let v = mysql_row.take::<chrono::NaiveDateTime, _>(i);
+                    v.map(|v| ScalarImpl::from(Timestamp::from(v)))
+                }
+                DataType::Timestamptz => {
+                    let v = mysql_row.take::<chrono::NaiveDateTime, _>(i);
+                    v.map(|v| ScalarImpl::from(Timestamptz::from_micros(v.timestamp_micros())))
+                }
+                DataType::Bytea => {
+                    let v = mysql_row.take::<Vec<u8>, _>(i);
+                    v.map(|v| ScalarImpl::from(v.into_boxed_slice()))
+                }
+                DataType::Jsonb => {
+                    let v = mysql_row.take::<serde_json::Value, _>(i);
+                    v.map(|v| ScalarImpl::from(JsonbVal::from(v)))
+                }
+                DataType::Interval
+                | DataType::Struct(_)
+                | DataType::List(_)
+                | DataType::Int256
+                | DataType::Serial => {
+                    // Interval, Struct, List, Int256 are not supported
+                    tracing::warn!(rw_field.name, ?rw_field.data_type, "unsupported data type, set to Null");
+                    None
+                }
+            }
+        };
+        datums.push(datum);
+    }
+    datums
+}
+
+#[cfg(test)]
+mod tests {
+
+    use futures::pin_mut;
+    use mysql_async::prelude::*;
+    use mysql_async::Row as MySqlRow;
+    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::row::{OwnedRow, Row};
+    use risingwave_common::types::{DataType, ToText};
+    use tokio_stream::StreamExt;
+
+    use crate::parser::mysql_row_to_datums;
+
+    // manual test case
+    #[ignore]
+    #[tokio::test]
+    async fn test_convert_mysql_row_to_owned_row() {
+        let pool = mysql_async::Pool::new("mysql://root:123456@localhost:8306/mydb");
+
+        let t1schema = Schema::new(vec![
+            Field::with_name(DataType::Int32, "v1"),
+            Field::with_name(DataType::Int32, "v2"),
+            Field::with_name(DataType::Timestamptz, "v3"),
+        ]);
+
+        let mut conn = pool.get_conn().await.unwrap();
+        conn.exec_drop("SET time_zone = \"+08:00\"", ())
+            .await
+            .unwrap();
+
+        let mut result_set = conn.query_iter("SELECT * FROM `t1m`").await.unwrap();
+        let s = result_set.stream::<MySqlRow>().await.unwrap().unwrap();
+        let row_stream = s.map(|row| {
+            // convert mysql row into OwnedRow
+            let mut mysql_row = row.unwrap();
+            let datums = mysql_row_to_datums(&mut mysql_row, &t1schema);
+            Ok::<_, anyhow::Error>(Some(OwnedRow::new(datums)))
+        });
+        pin_mut!(row_stream);
+        while let Some(row) = row_stream.next().await {
+            if let Ok(ro) = row && ro.is_some() {
+                let owned_row = ro.unwrap();
+                let d = owned_row.datum_at(2);
+                if let Some(scalar) = d {
+                    let v = scalar.into_timestamptz();
+                    println!("timestamp: {}", v.to_text());
+                }
+            }
+        }
+    }
+}
diff --git a/src/connector/src/parser/protobuf/.gitignore b/src/connector/src/parser/protobuf/.gitignore
new file mode 100644
index 0000000000000..4109deeeb3337
--- /dev/null
+++ b/src/connector/src/parser/protobuf/.gitignore
@@ -0,0 +1 @@
+recursive.rs
diff --git a/src/connector/src/parser/protobuf/mod.rs b/src/connector/src/parser/protobuf/mod.rs
index 8870ee8f67b48..a2874e325e1fd 100644
--- a/src/connector/src/parser/protobuf/mod.rs
+++ b/src/connector/src/parser/protobuf/mod.rs
@@ -15,3 +15,7 @@
 mod parser;
 pub use parser::*;
 mod schema_resolver;
+
+#[rustfmt::skip]
+#[cfg(test)]
+mod recursive;
diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs
index d7808a6c4e371..09763bcd6a49d 100644
--- a/src/connector/src/parser/protobuf/parser.rs
+++ b/src/connector/src/parser/protobuf/parser.rs
@@ -25,11 +25,12 @@ use risingwave_common::error::{Result, RwError};
 use risingwave_common::try_match_expand;
 use risingwave_common::types::{DataType, Datum, Decimal, ScalarImpl, F32, F64};
 use risingwave_pb::plan_common::ColumnDesc;
-use url::Url;
 
 use super::schema_resolver::*;
 use crate::aws_utils::load_file_descriptor_from_s3;
-use crate::parser::schema_registry::{extract_schema_id, get_subject_by_strategy, Client};
+use crate::parser::schema_registry::{
+    extract_schema_id, get_subject_by_strategy, handle_sr_list, Client,
+};
 use crate::parser::unified::protobuf::ProtobufAccess;
 use crate::parser::unified::AccessImpl;
 use crate::parser::{AccessBuilder, EncodingProperties};
@@ -80,8 +81,7 @@ impl ProtobufParserConfig {
         let protobuf_config = try_match_expand!(encoding_properties, EncodingProperties::Protobuf)?;
         let location = &protobuf_config.row_schema_location;
         let message_name = &protobuf_config.message_name;
-        let url = Url::parse(location)
-            .map_err(|e| InternalError(format!("failed to parse url ({}): {}", location, e)))?;
+        let url = handle_sr_list(location.as_str())?;
 
         let schema_bytes = if protobuf_config.use_schema_registry {
             let (schema_key, schema_value) = get_subject_by_strategy(
@@ -99,6 +99,7 @@ impl ProtobufParserConfig {
             let client = Client::new(url, &protobuf_config.client_config)?;
             compile_file_descriptor_from_schema_registry(schema_value.as_str(), &client).await?
         } else {
+            let url = url.get(0).unwrap();
             match url.scheme() {
                 // TODO(Tao): support local file only when it's compiled in debug mode.
                 "file" => {
@@ -115,12 +116,12 @@ impl ProtobufParserConfig {
                 }
                 "s3" => {
                     load_file_descriptor_from_s3(
-                        &url,
+                        url,
                         protobuf_config.aws_auth_props.as_ref().unwrap(),
                     )
                     .await
                 }
-                "https" | "http" => load_file_descriptor_from_http(&url).await,
+                "https" | "http" => load_file_descriptor_from_http(url).await,
                 scheme => Err(RwError::from(ProtocolError(format!(
                     "path scheme {} is not supported",
                     scheme
@@ -299,11 +300,12 @@ fn protobuf_type_mapping(
         Kind::Bool => DataType::Boolean,
         Kind::Double => DataType::Float64,
         Kind::Float => DataType::Float32,
-        Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 | Kind::Fixed32 => DataType::Int32,
-        Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 | Kind::Fixed64 | Kind::Uint32 => {
+        Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 => DataType::Int32,
+        // Fixed32 represents [0, 2^32 - 1]. It's equal to u32.
+        Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 | Kind::Uint32 | Kind::Fixed32 => {
             DataType::Int64
         }
-        Kind::Uint64 => DataType::Decimal,
+        Kind::Uint64 | Kind::Fixed64 => DataType::Decimal,
         Kind::String => DataType::Varchar,
         Kind::Message(m) => {
             let fields = m
@@ -316,6 +318,12 @@ fn protobuf_type_mapping(
         Kind::Enum(_) => DataType::Varchar,
         Kind::Bytes => DataType::Bytea,
     };
+    if field_descriptor.is_map() {
+        return Err(RwError::from(ProtocolError(format!(
+            "map type is unsupported (field: '{}')",
+            field_descriptor.full_name()
+        ))));
+    }
     if field_descriptor.cardinality() == Cardinality::Repeated {
         t = DataType::List(Box::new(t))
     }
@@ -341,15 +349,18 @@ pub(crate) fn resolve_pb_header(payload: &[u8]) -> Result<&[u8]> {
 
 #[cfg(test)]
 mod test {
-
     use std::collections::HashMap;
     use std::path::PathBuf;
 
-    use bytes::Bytes;
+    use prost::Message;
+    use risingwave_common::types::{DataType, StructType};
     use risingwave_pb::catalog::StreamSourceInfo;
     use risingwave_pb::data::data_type::PbTypeName;
 
     use super::*;
+    use crate::parser::protobuf::recursive::all_types::{EnumType, ExampleOneof, NestedMessage};
+    use crate::parser::protobuf::recursive::AllTypes;
+    use crate::parser::unified::Access;
     use crate::parser::SpecificParserConfig;
     use crate::source::{SourceEncode, SourceFormat, SourceStruct};
 
@@ -497,8 +508,7 @@ mod test {
         assert!(columns.is_err());
     }
 
-    #[tokio::test]
-    async fn test_all_types() {
+    async fn create_recursive_pb_parser_config() -> ProtobufParserConfig {
         let location = schema_dir() + "/proto_recursive/recursive.pb";
         let message_name = "recursive.AllTypes";
 
@@ -514,19 +524,209 @@ mod test {
             &HashMap::new(),
         )
         .unwrap();
-        let conf = ProtobufParserConfig::new(parser_config.encoding_config)
+
+        ProtobufParserConfig::new(parser_config.encoding_config)
             .await
-            .unwrap();
-        // Ensure that the parser can recognize the schema.
-        conf.map_to_columns().unwrap();
+            .unwrap()
+    }
 
-        let field_desc = conf
-            .message_descriptor
-            .get_field_by_name("bytes_field")
-            .unwrap();
-        let d = from_protobuf_value(&field_desc, &Value::Bytes(Bytes::from(vec![1, 2, 3])))
+    #[tokio::test]
+    async fn test_all_types_create_source() {
+        let conf = create_recursive_pb_parser_config().await;
+
+        // Ensure that the parser can recognize the schema.
+        let columns = conf
+            .map_to_columns()
             .unwrap()
-            .unwrap();
-        assert_eq!(d, ScalarImpl::Bytea(vec![1, 2, 3].into_boxed_slice()));
+            .into_iter()
+            .map(|c| DataType::from(&c.column_type.unwrap()))
+            .collect_vec();
+        assert_eq!(
+            columns,
+            vec![
+                DataType::Float64, // double_field
+                DataType::Float32, // float_field
+                DataType::Int32,   // int32_field
+                DataType::Int64,   // int64_field
+                DataType::Int64,   // uint32_field
+                DataType::Decimal, // uint64_field
+                DataType::Int32,   // sint32_field
+                DataType::Int64,   // sint64_field
+                DataType::Int64,   // fixed32_field
+                DataType::Decimal, // fixed64_field
+                DataType::Int32,   // sfixed32_field
+                DataType::Int64,   // sfixed64_field
+                DataType::Boolean, // bool_field
+                DataType::Varchar, // string_field
+                DataType::Bytea,   // bytes_field
+                DataType::Varchar, // enum_field
+                DataType::Struct(StructType::new(vec![
+                    ("id", DataType::Int32),
+                    ("name", DataType::Varchar)
+                ])), // nested_message_field
+                DataType::List(DataType::Int32.into()), // repeated_int_field
+                DataType::Varchar, // oneof_string
+                DataType::Int32,   // oneof_int32
+                DataType::Varchar, // oneof_enum
+                DataType::Struct(StructType::new(vec![
+                    ("seconds", DataType::Int64),
+                    ("nanos", DataType::Int32)
+                ])), // timestamp_field
+                DataType::Struct(StructType::new(vec![
+                    ("seconds", DataType::Int64),
+                    ("nanos", DataType::Int32)
+                ])), // duration_field
+                DataType::Struct(StructType::new(vec![
+                    ("type_url", DataType::Varchar),
+                    ("value", DataType::Bytea),
+                ])), // any_field
+                DataType::Struct(StructType::new(vec![("value", DataType::Int32)])), /* int32_value_field */
+                DataType::Struct(StructType::new(vec![("value", DataType::Varchar)])), /* string_value_field */
+            ]
+        )
+    }
+
+    #[tokio::test]
+    async fn test_all_types_data_parsing() {
+        let m = create_all_types_message();
+        let mut payload = Vec::new();
+        m.encode(&mut payload).unwrap();
+
+        let conf = create_recursive_pb_parser_config().await;
+        let mut access_builder = ProtobufAccessBuilder::new(conf).unwrap();
+        let access = access_builder.generate_accessor(payload).await.unwrap();
+        if let AccessImpl::Protobuf(a) = access {
+            assert_all_types_eq(&a, &m);
+        } else {
+            panic!("unexpected")
+        }
+    }
+
+    fn assert_all_types_eq(a: &ProtobufAccess, m: &AllTypes) {
+        type S = ScalarImpl;
+
+        pb_eq(a, "double_field", S::Float64(m.double_field.into()));
+        pb_eq(a, "float_field", S::Float32(m.float_field.into()));
+        pb_eq(a, "int32_field", S::Int32(m.int32_field));
+        pb_eq(a, "int64_field", S::Int64(m.int64_field));
+        pb_eq(a, "uint32_field", S::Int64(m.uint32_field.into()));
+        pb_eq(a, "uint64_field", S::Decimal(m.uint64_field.into()));
+        pb_eq(a, "sint32_field", S::Int32(m.sint32_field));
+        pb_eq(a, "sint64_field", S::Int64(m.sint64_field));
+        pb_eq(a, "fixed32_field", S::Int64(m.fixed32_field.into()));
+        pb_eq(a, "fixed64_field", S::Decimal(m.fixed64_field.into()));
+        pb_eq(a, "sfixed32_field", S::Int32(m.sfixed32_field));
+        pb_eq(a, "sfixed64_field", S::Int64(m.sfixed64_field));
+        pb_eq(a, "bool_field", S::Bool(m.bool_field));
+        pb_eq(a, "string_field", S::Utf8(m.string_field.as_str().into()));
+        pb_eq(a, "bytes_field", S::Bytea(m.bytes_field.clone().into()));
+        pb_eq(a, "enum_field", S::Utf8("OPTION1".into()));
+        pb_eq(
+            a,
+            "nested_message_field",
+            S::Struct(StructValue::new(vec![
+                Some(ScalarImpl::Int32(100)),
+                Some(ScalarImpl::Utf8("Nested".into())),
+            ])),
+        );
+        pb_eq(
+            a,
+            "repeated_int_field",
+            S::List(ListValue::new(
+                m.repeated_int_field
+                    .iter()
+                    .map(|&x| Some(x.into()))
+                    .collect(),
+            )),
+        );
+        pb_eq(
+            a,
+            "timestamp_field",
+            S::Struct(StructValue::new(vec![
+                Some(ScalarImpl::Int64(1630927032)),
+                Some(ScalarImpl::Int32(500000000)),
+            ])),
+        );
+        pb_eq(
+            a,
+            "duration_field",
+            S::Struct(StructValue::new(vec![
+                Some(ScalarImpl::Int64(60)),
+                Some(ScalarImpl::Int32(500000000)),
+            ])),
+        );
+        pb_eq(
+            a,
+            "any_field",
+            S::Struct(StructValue::new(vec![
+                Some(ScalarImpl::Utf8(
+                    m.any_field.as_ref().unwrap().type_url.as_str().into(),
+                )),
+                Some(ScalarImpl::Bytea(
+                    m.any_field.as_ref().unwrap().value.clone().into(),
+                )),
+            ])),
+        );
+        pb_eq(
+            a,
+            "int32_value_field",
+            S::Struct(StructValue::new(vec![Some(ScalarImpl::Int32(42))])),
+        );
+        pb_eq(
+            a,
+            "string_value_field",
+            S::Struct(StructValue::new(vec![Some(ScalarImpl::Utf8(
+                m.string_value_field.as_ref().unwrap().as_str().into(),
+            ))])),
+        );
+        pb_eq(a, "oneof_string", S::Utf8("".into()));
+        pb_eq(a, "oneof_int32", S::Int32(123));
+        pb_eq(a, "oneof_enum", S::Utf8("DEFAULT".into()));
+    }
+
+    fn pb_eq(a: &ProtobufAccess, field_name: &str, value: ScalarImpl) {
+        let d = a.access(&[field_name], None).unwrap().unwrap();
+        assert_eq!(d, value, "field: {} value: {:?}", field_name, d);
+    }
+
+    fn create_all_types_message() -> AllTypes {
+        AllTypes {
+            double_field: 1.2345,
+            float_field: 1.2345,
+            int32_field: 42,
+            int64_field: 1234567890,
+            uint32_field: 98765,
+            uint64_field: 9876543210,
+            sint32_field: -12345,
+            sint64_field: -987654321,
+            fixed32_field: 1234,
+            fixed64_field: 5678,
+            sfixed32_field: -56789,
+            sfixed64_field: -123456,
+            bool_field: true,
+            string_field: "Hello, Prost!".to_string(),
+            bytes_field: b"byte data".to_vec(),
+            enum_field: EnumType::Option1 as i32,
+            nested_message_field: Some(NestedMessage {
+                id: 100,
+                name: "Nested".to_string(),
+            }),
+            repeated_int_field: vec![1, 2, 3, 4, 5],
+            timestamp_field: Some(::prost_types::Timestamp {
+                seconds: 1630927032,
+                nanos: 500000000,
+            }),
+            duration_field: Some(::prost_types::Duration {
+                seconds: 60,
+                nanos: 500000000,
+            }),
+            any_field: Some(::prost_types::Any {
+                type_url: "type.googleapis.com/my_custom_type".to_string(),
+                value: b"My custom data".to_vec(),
+            }),
+            int32_value_field: Some(42),
+            string_value_field: Some("Hello, Wrapper!".to_string()),
+            example_oneof: Some(ExampleOneof::OneofInt32(123)),
+        }
     }
 }
diff --git a/src/connector/src/parser/schema_registry/client.rs b/src/connector/src/parser/schema_registry/client.rs
index ba9b6eb7dbac2..31fc300d7e0c4 100644
--- a/src/connector/src/parser/schema_registry/client.rs
+++ b/src/connector/src/parser/schema_registry/client.rs
@@ -13,31 +13,46 @@
 // limitations under the License.
 
 use std::collections::HashSet;
+use std::fmt::Debug;
+use std::sync::Arc;
 
+use futures::future::select_all;
+use itertools::Itertools;
 use reqwest::{Method, Url};
 use risingwave_common::error::ErrorCode::ProtocolError;
 use risingwave_common::error::{Result, RwError};
 use serde::de::DeserializeOwned;
-use serde::{Deserialize, Serialize};
 
+use crate::parser::schema_registry::util::*;
 use crate::parser::SchemaRegistryAuth;
 
 /// An client for communication with schema registry
 #[derive(Debug)]
 pub struct Client {
     inner: reqwest::Client,
-    url: Url,
+    url: Vec<Url>,
     username: Option<String>,
     password: Option<String>,
 }
 
 impl Client {
-    pub(crate) fn new(url: Url, client_config: &SchemaRegistryAuth) -> Result<Self> {
-        if url.cannot_be_a_base() {
+    pub(crate) fn new(url: Vec<Url>, client_config: &SchemaRegistryAuth) -> Result<Self> {
+        let valid_urls = url
+            .iter()
+            .map(|url| (url.cannot_be_a_base(), url))
+            .filter(|(x, _)| !*x)
+            .map(|(_, url)| url.clone())
+            .collect_vec();
+        if valid_urls.is_empty() {
             return Err(RwError::from(ProtocolError(format!(
-                "{} cannot be a base url",
+                "no valid url provided, got {:?}",
                 url
             ))));
+        } else {
+            tracing::debug!(
+                "schema registry client will use url {:?} to connect",
+                valid_urls
+            );
         }
 
         let inner = reqwest::Client::builder().build().map_err(|e| {
@@ -46,36 +61,60 @@ impl Client {
 
         Ok(Client {
             inner,
-            url,
+            url: valid_urls,
             username: client_config.username.clone(),
             password: client_config.password.clone(),
         })
     }
 
-    fn build_request<P>(&self, method: Method, path: P) -> reqwest::RequestBuilder
+    async fn concurrent_req<'a, T>(
+        &'a self,
+        method: Method,
+        path: &'a [&'a (impl AsRef<str> + ?Sized + Debug + ToString)],
+    ) -> Result<T>
     where
-        P: IntoIterator,
-        P::Item: AsRef<str>,
+        T: DeserializeOwned + Send + Sync + 'static,
     {
-        let mut url = self.url.clone();
-        url.path_segments_mut()
-            .expect("constructor validated URL can be a base")
-            .clear()
-            .extend(path);
-
-        let mut request = self.inner.request(method, url);
+        let mut fut_req = Vec::with_capacity(self.url.len());
+        let mut errs = Vec::with_capacity(self.url.len());
+        let ctx = Arc::new(SchemaRegistryCtx {
+            username: self.username.clone(),
+            password: self.password.clone(),
+            client: self.inner.clone(),
+            path: path.iter().map(|p| p.to_string()).collect_vec(),
+        });
+        for url in &self.url {
+            fut_req.push(tokio::spawn(req_inner(
+                ctx.clone(),
+                url.clone(),
+                method.clone(),
+            )));
+        }
 
-        if self.username.is_some() {
-            request = request.basic_auth(self.username.clone().unwrap(), self.password.clone())
+        while !fut_req.is_empty() {
+            let (result, _index, remaining) = select_all(fut_req).await;
+            match result {
+                Ok(Ok(res)) => {
+                    let _ = remaining.iter().map(|ele| ele.abort());
+                    return Ok(res);
+                }
+                Ok(Err(e)) => errs.push(e),
+                Err(e) => errs.push(RwError::from(e)),
+            }
+            fut_req = remaining;
         }
 
-        request
+        Err(RwError::from(ProtocolError(format!(
+            "all request confluent registry all timeout, req path {:?}, urls {:?}, err: {:?}",
+            path, self.url, errs
+        ))))
     }
 
     /// get schema by id
     pub async fn get_schema_by_id(&self, id: i32) -> Result<ConfluentSchema> {
-        let req = self.build_request(Method::GET, &["schemas", "ids", &id.to_string()]);
-        let res: GetByIdResp = request(req).await?;
+        let res: GetByIdResp = self
+            .concurrent_req(Method::GET, &["schemas", "ids", &id.to_string()])
+            .await?;
         Ok(ConfluentSchema {
             id,
             content: res.schema,
@@ -89,8 +128,9 @@ impl Client {
 
     /// get the latest version of the subject
     pub async fn get_subject(&self, subject: &str) -> Result<Subject> {
-        let req = self.build_request(Method::GET, &["subjects", subject, "versions", "latest"]);
-        let res: GetBySubjectResp = request(req).await?;
+        let res: GetBySubjectResp = self
+            .concurrent_req(Method::GET, &["subjects", subject, "versions", "latest"])
+            .await?;
         tracing::debug!("update schema: {:?}", res);
         Ok(Subject {
             schema: ConfluentSchema {
@@ -112,9 +152,9 @@ impl Client {
         let mut queue = vec![(subject.to_owned(), "latest".to_owned())];
         // use bfs to get all references
         while let Some((subject, version)) = queue.pop() {
-            let req =
-                self.build_request(Method::GET, &["subjects", &subject, "versions", &version]);
-            let res: GetBySubjectResp = request(req).await?;
+            let res: GetBySubjectResp = self
+                .concurrent_req(Method::GET, &["subjects", &subject, "versions", &version])
+                .await?;
             let ref_subject = Subject {
                 schema: ConfluentSchema {
                     id: res.id,
@@ -138,98 +178,6 @@ impl Client {
     }
 }
 
-async fn request<T>(req: reqwest::RequestBuilder) -> Result<T>
-where
-    T: DeserializeOwned,
-{
-    let res = req.send().await.map_err(|e| {
-        RwError::from(ProtocolError(format!(
-            "confluent registry send req error {}",
-            e
-        )))
-    })?;
-    let status = res.status();
-    if status.is_success() {
-        res.json().await.map_err(|e| {
-            RwError::from(ProtocolError(format!(
-                "confluent registry parse resp error {}",
-                e
-            )))
-        })
-    } else {
-        let res = res.json::<ErrorResp>().await.map_err(|e| {
-            RwError::from(ProtocolError(format!(
-                "confluent registry resp error {}",
-                e
-            )))
-        })?;
-        Err(RwError::from(ProtocolError(format!(
-            "confluent registry resp error, code: {}, msg {}",
-            res.error_code, res.message
-        ))))
-    }
-}
-
-/// `Schema` format of confluent schema registry
-#[derive(Debug, Eq, PartialEq)]
-pub struct ConfluentSchema {
-    /// The id of the schema
-    pub id: i32,
-    /// The raw text of the schema def
-    pub content: String,
-}
-
-/// `Subject` stored in confluent schema registry
-#[derive(Debug, Eq, PartialEq)]
-pub struct Subject {
-    /// The version of the current schema
-    pub version: i32,
-    /// The name of the schema
-    pub name: String,
-    /// The schema corresponding to that `version`
-    pub schema: ConfluentSchema,
-}
-
-/// One schema can reference another schema
-/// (e.g., import "other.proto" in protobuf)
-#[derive(Debug, Serialize, Deserialize)]
-pub struct SchemaReference {
-    /// The name of the reference.
-    pub name: String,
-    /// The subject that the referenced schema belongs to
-    pub subject: String,
-    /// The version of the referenced schema
-    pub version: i32,
-}
-
-#[derive(Debug, Deserialize)]
-struct GetByIdResp {
-    schema: String,
-}
-
-#[derive(Debug, Deserialize)]
-struct GetBySubjectResp {
-    id: i32,
-    schema: String,
-    version: i32,
-    subject: String,
-    // default to empty/non-reference
-    #[serde(default)]
-    references: Vec<SchemaReference>,
-}
-
-#[derive(Debug, Deserialize)]
-struct ErrorResp {
-    error_code: i32,
-    message: String,
-}
-
-#[derive(Debug)]
-enum ReqResp<T> {
-    Succeed(T),
-    Failed(ErrorResp),
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -239,7 +187,7 @@ mod tests {
     async fn test_get_subject() {
         let url = Url::parse("http://localhost:8081").unwrap();
         let client = Client::new(
-            url,
+            vec![url],
             &SchemaRegistryAuth {
                 username: None,
                 password: None,
diff --git a/src/connector/src/parser/schema_registry/util.rs b/src/connector/src/parser/schema_registry/util.rs
index b4309a40f7ead..95d4dc5e767ae 100644
--- a/src/connector/src/parser/schema_registry/util.rs
+++ b/src/connector/src/parser/schema_registry/util.rs
@@ -12,9 +12,40 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Debug;
+use std::sync::Arc;
+
 use byteorder::{BigEndian, ByteOrder};
-use risingwave_common::error::ErrorCode::InternalError;
+use reqwest::Method;
+use risingwave_common::error::ErrorCode::{InternalError, ProtocolError};
 use risingwave_common::error::{Result, RwError};
+use serde::de::DeserializeOwned;
+use serde_derive::{Deserialize, Serialize};
+use url::{ParseError, Url};
+
+pub fn handle_sr_list(addr: &str) -> Result<Vec<Url>> {
+    let segment = addr.split(',').collect::<Vec<&str>>();
+    let mut errs: Vec<ParseError> = Vec::with_capacity(segment.len());
+    let mut urls = Vec::with_capacity(segment.len());
+    for ele in segment {
+        match ele.parse::<Url>() {
+            Ok(url) => urls.push(url),
+            Err(e) => errs.push(e),
+        }
+    }
+    if urls.is_empty() {
+        return Err(RwError::from(ProtocolError(format!(
+            "no valid url provided, got {:?}",
+            errs
+        ))));
+    }
+    tracing::debug!(
+        "schema registry client will use url {:?} to connect, the rest failed because: {:?}",
+        urls,
+        errs
+    );
+    Ok(urls)
+}
 
 /// extract the magic number and `schema_id` at the front of payload
 ///
@@ -41,3 +72,155 @@ pub(crate) fn extract_schema_id(payload: &[u8]) -> Result<(i32, &[u8])> {
 
     Ok((schema_id, &payload[header_len..]))
 }
+
+pub(crate) struct SchemaRegistryCtx {
+    pub username: Option<String>,
+    pub password: Option<String>,
+    pub client: reqwest::Client,
+    pub path: Vec<String>,
+}
+
+pub(crate) async fn req_inner<T>(
+    ctx: Arc<SchemaRegistryCtx>,
+    mut url: Url,
+    method: Method,
+) -> Result<T>
+where
+    T: DeserializeOwned + Send + Sync + 'static,
+{
+    url.path_segments_mut()
+        .expect("constructor validated URL can be a base")
+        .clear()
+        .extend(&ctx.path);
+    tracing::debug!("request to url: {}, method {}", &url, &method);
+    let mut request_builder = ctx.client.request(method, url);
+
+    if let Some(ref username) = ctx.username {
+        request_builder = request_builder.basic_auth(username, ctx.password.as_ref());
+    }
+    request(request_builder).await
+}
+
+async fn request<T>(req: reqwest::RequestBuilder) -> Result<T>
+where
+    T: DeserializeOwned,
+{
+    let res = req.send().await.map_err(|e| {
+        RwError::from(ProtocolError(format!(
+            "confluent registry send req error {}",
+            e
+        )))
+    })?;
+    let status = res.status();
+    if status.is_success() {
+        res.json().await.map_err(|e| {
+            RwError::from(ProtocolError(format!(
+                "confluent registry parse resp error {}",
+                e
+            )))
+        })
+    } else {
+        let res = res.json::<ErrorResp>().await.map_err(|e| {
+            RwError::from(ProtocolError(format!(
+                "confluent registry resp error {}",
+                e
+            )))
+        })?;
+        Err(RwError::from(ProtocolError(format!(
+            "confluent registry resp error, code: {}, msg {}",
+            res.error_code, res.message
+        ))))
+    }
+}
+
+/// `Schema` format of confluent schema registry
+#[derive(Debug, Eq, PartialEq)]
+pub struct ConfluentSchema {
+    /// The id of the schema
+    pub id: i32,
+    /// The raw text of the schema def
+    pub content: String,
+}
+
+/// `Subject` stored in confluent schema registry
+#[derive(Debug, Eq, PartialEq)]
+pub struct Subject {
+    /// The version of the current schema
+    pub version: i32,
+    /// The name of the schema
+    pub name: String,
+    /// The schema corresponding to that `version`
+    pub schema: ConfluentSchema,
+}
+
+/// One schema can reference another schema
+/// (e.g., import "other.proto" in protobuf)
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SchemaReference {
+    /// The name of the reference.
+    pub name: String,
+    /// The subject that the referenced schema belongs to
+    pub subject: String,
+    /// The version of the referenced schema
+    pub version: i32,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct GetByIdResp {
+    pub schema: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct GetBySubjectResp {
+    pub id: i32,
+    pub schema: String,
+    pub version: i32,
+    pub subject: String,
+    // default to empty/non-reference
+    #[serde(default)]
+    pub references: Vec<SchemaReference>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ErrorResp {
+    error_code: i32,
+    message: String,
+}
+
+#[derive(Debug)]
+enum ReqResp<T> {
+    Succeed(T),
+    Failed(ErrorResp),
+}
+
+#[cfg(test)]
+mod test {
+    use crate::parser::schema_registry::handle_sr_list;
+
+    #[test]
+    fn test_handle_sr_list() {
+        let addr1 = "http://localhost:8081".to_owned();
+        assert_eq!(
+            handle_sr_list(&addr1).unwrap(),
+            vec!["http://localhost:8081".parse().unwrap()]
+        );
+
+        let addr2 = "http://localhost:8081,http://localhost:8082".to_owned();
+        assert_eq!(
+            handle_sr_list(&addr2).unwrap(),
+            vec![
+                "http://localhost:8081".parse().unwrap(),
+                "http://localhost:8082".parse().unwrap()
+            ]
+        );
+
+        let fail_addr = "http://localhost:8081,12345".to_owned();
+        assert_eq!(
+            handle_sr_list(&fail_addr).unwrap(),
+            vec!["http://localhost:8081".parse().unwrap(),]
+        );
+
+        let all_fail_addr = "54321,12345".to_owned();
+        assert!(handle_sr_list(&all_fail_addr).is_err());
+    }
+}
diff --git a/src/connector/src/parser/unified/avro.rs b/src/connector/src/parser/unified/avro.rs
index ea9841dd981c0..3b7bcd14e7b6c 100644
--- a/src/connector/src/parser/unified/avro.rs
+++ b/src/connector/src/parser/unified/avro.rs
@@ -396,7 +396,7 @@ pub fn avro_extract_field_schema<'a>(
             let name =
                 name.ok_or_else(|| anyhow::format_err!("no name provided for a field in record"))?;
             let index = lookup.get(name).ok_or_else(|| {
-                anyhow::format_err!("No filed named {} in record {:?}", name, schema)
+                anyhow::format_err!("no field named '{}' in record: {:?}", name, schema)
             })?;
             let field = fields
                 .get(*index)
diff --git a/src/connector/src/parser/util.rs b/src/connector/src/parser/util.rs
index ff2d201e8b025..7444fe202de46 100644
--- a/src/connector/src/parser/util.rs
+++ b/src/connector/src/parser/util.rs
@@ -12,15 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::collections::HashMap;
+use std::path::Path;
 
 use bytes::Bytes;
 use itertools::Itertools;
 use reqwest::Url;
-use risingwave_common::error::ErrorCode::{InternalError, InvalidParameterValue, ProtocolError};
+use risingwave_common::error::ErrorCode::{
+    InternalError, InvalidConfigValue, InvalidParameterValue, ProtocolError,
+};
 use risingwave_common::error::{Result, RwError};
 
+use crate::aws_auth::AwsAuthProps;
+use crate::aws_utils::{default_conn_config, s3_client};
 use crate::parser::WriteGuard;
 
+const AVRO_SCHEMA_LOCATION_S3_REGION: &str = "region";
+
 /// get kafka topic name
 pub(super) fn get_kafka_topic(props: &HashMap<String, String>) -> Result<&String> {
     const KAFKA_TOPIC_KEY1: &str = "kafka.topic";
@@ -118,3 +125,53 @@ macro_rules! extract_key_config {
         }
     };
 }
+
+/// Read schema from local file. For on-premise or testing.
+pub(super) fn read_schema_from_local(path: impl AsRef<Path>) -> Result<String> {
+    std::fs::read_to_string(path.as_ref()).map_err(|e| e.into())
+}
+
+/// Read schema from http/https. For common usage.
+pub(super) async fn read_schema_from_http(location: &Url) -> Result<String> {
+    let schema_bytes = download_from_http(location).await?;
+
+    String::from_utf8(schema_bytes.into()).map_err(|e| {
+        RwError::from(InternalError(format!(
+            "read schema string from https failed {}",
+            e
+        )))
+    })
+}
+
+/// Read schema from s3 bucket.
+/// S3 file location format: <s3://bucket_name/file_name>
+pub(super) async fn read_schema_from_s3(url: &Url, config: &AwsAuthProps) -> Result<String> {
+    let bucket = url
+        .domain()
+        .ok_or_else(|| RwError::from(InternalError(format!("Illegal s3 path {}", url))))?;
+    if config.region.is_none() {
+        return Err(RwError::from(InvalidConfigValue {
+            config_entry: AVRO_SCHEMA_LOCATION_S3_REGION.to_string(),
+            config_value: "NONE".to_string(),
+        }));
+    }
+    let key = url.path().replace('/', "");
+    let sdk_config = config.build_config().await?;
+    let s3_client = s3_client(&sdk_config, Some(default_conn_config()));
+    let response = s3_client
+        .get_object()
+        .bucket(bucket.to_string())
+        .key(key)
+        .send()
+        .await
+        .map_err(|e| RwError::from(InternalError(e.to_string())))?;
+    let body_bytes = response.body.collect().await.map_err(|e| {
+        RwError::from(InternalError(format!(
+            "Read Avro schema file from s3 {}",
+            e
+        )))
+    })?;
+    let schema_bytes = body_bytes.into_bytes().to_vec();
+    String::from_utf8(schema_bytes)
+        .map_err(|e| RwError::from(InternalError(format!("Avro schema not valid utf8 {}", e))))
+}
diff --git a/src/connector/src/sink/boxed.rs b/src/connector/src/sink/boxed.rs
index 4b3176dec7282..0a7082961a00e 100644
--- a/src/connector/src/sink/boxed.rs
+++ b/src/connector/src/sink/boxed.rs
@@ -14,6 +14,7 @@
 
 use std::fmt::{Debug, Formatter};
 use std::ops::{Deref, DerefMut};
+use std::sync::Arc;
 
 use async_trait::async_trait;
 use risingwave_common::array::StreamChunk;
@@ -54,7 +55,7 @@ impl<CM: 'static + Send> SinkWriter for BoxWriter<CM> {
         self.deref_mut().abort().await
     }
 
-    async fn update_vnode_bitmap(&mut self, vnode_bitmap: Bitmap) -> crate::sink::Result<()> {
+    async fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc<Bitmap>) -> crate::sink::Result<()> {
         self.deref_mut().update_vnode_bitmap(vnode_bitmap).await
     }
 }
diff --git a/src/connector/src/sink/catalog/desc.rs b/src/connector/src/sink/catalog/desc.rs
index 1e8aa6a068739..f00709188d23c 100644
--- a/src/connector/src/sink/catalog/desc.rs
+++ b/src/connector/src/sink/catalog/desc.rs
@@ -54,6 +54,13 @@ pub struct SinkDesc {
     // based on both its own derivation on the append-only attribute and other user-specified
     // options in `properties`.
     pub sink_type: SinkType,
+
+    /// Name of the database
+    pub db_name: String,
+
+    /// Name of the "table" field for Debezium. If the sink is from table or mv,
+    /// it is the name of table/mv. Otherwise, it is the name of the sink.
+    pub sink_from_name: String,
 }
 
 impl SinkDesc {
@@ -82,6 +89,8 @@ impl SinkDesc {
             connection_id,
             created_at_epoch: None,
             initialized_at_epoch: None,
+            db_name: self.db_name,
+            sink_from_name: self.sink_from_name,
         }
     }
 
@@ -100,6 +109,8 @@ impl SinkDesc {
             distribution_key: self.distribution_key.iter().map(|k| *k as _).collect_vec(),
             properties: self.properties.clone().into_iter().collect(),
             sink_type: self.sink_type.to_proto() as i32,
+            db_name: self.db_name.clone(),
+            sink_from_name: self.sink_from_name.clone(),
         }
     }
 }
diff --git a/src/connector/src/sink/catalog/mod.rs b/src/connector/src/sink/catalog/mod.rs
index 5cd291dc0ae75..16b53af17499a 100644
--- a/src/connector/src/sink/catalog/mod.rs
+++ b/src/connector/src/sink/catalog/mod.rs
@@ -150,6 +150,12 @@ pub struct SinkCatalog {
     pub created_at_epoch: Option<Epoch>,
 
     pub initialized_at_epoch: Option<Epoch>,
+
+    /// Name of the database
+    pub db_name: String,
+
+    /// Name for the table info for Debezium sink
+    pub sink_from_name: String,
 }
 
 impl SinkCatalog {
@@ -183,6 +189,8 @@ impl SinkCatalog {
             connection_id: self.connection_id.map(|id| id.into()),
             initialized_at_epoch: self.initialized_at_epoch.map(|e| e.0),
             created_at_epoch: self.created_at_epoch.map(|e| e.0),
+            db_name: self.db_name.clone(),
+            sink_from_name: self.sink_from_name.clone(),
         }
     }
 
@@ -257,6 +265,8 @@ impl From<PbSink> for SinkCatalog {
             connection_id: pb.connection_id.map(ConnectionId),
             created_at_epoch: pb.created_at_epoch.map(Epoch::from),
             initialized_at_epoch: pb.initialized_at_epoch.map(Epoch::from),
+            db_name: pb.db_name,
+            sink_from_name: pb.sink_from_name,
         }
     }
 }
diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs
index e43444569b610..187b87397dbf4 100644
--- a/src/connector/src/sink/clickhouse.rs
+++ b/src/connector/src/sink/clickhouse.rs
@@ -19,7 +19,6 @@ use anyhow::anyhow;
 use clickhouse::{Client, Row as ClickHouseRow};
 use itertools::Itertools;
 use risingwave_common::array::{Op, RowRef, StreamChunk};
-use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::row::Row;
 use risingwave_common::types::{DataType, ScalarRefImpl, Serial};
@@ -439,14 +438,6 @@ impl SinkWriter for ClickHouseSinkWriter {
     async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
         Ok(())
     }
-
-    async fn abort(&mut self) -> Result<()> {
-        Ok(())
-    }
-
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        Ok(())
-    }
 }
 
 #[derive(ClickHouseRow, Deserialize)]
diff --git a/src/connector/src/sink/coordinate.rs b/src/connector/src/sink/coordinate.rs
index f81a2fe1bac3b..bbcc7b636b17c 100644
--- a/src/connector/src/sink/coordinate.rs
+++ b/src/connector/src/sink/coordinate.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use anyhow::anyhow;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
@@ -81,7 +83,7 @@ impl<W: SinkWriter<CommitMetadata = Option<SinkMetadata>>> SinkWriter for Coordi
         self.inner.abort().await
     }
 
-    async fn update_vnode_bitmap(&mut self, vnode_bitmap: Bitmap) -> Result<()> {
+    async fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc<Bitmap>) -> Result<()> {
         self.inner.update_vnode_bitmap(vnode_bitmap).await
     }
 }
diff --git a/src/connector/src/sink/iceberg.rs b/src/connector/src/sink/iceberg.rs
index b71af7c4bf214..451c8b2686ec7 100644
--- a/src/connector/src/sink/iceberg.rs
+++ b/src/connector/src/sink/iceberg.rs
@@ -411,12 +411,6 @@ impl SinkWriter for IcebergWriter {
         // TODO: abort should clean up all the data written in this epoch.
         Ok(())
     }
-
-    /// Update the vnode bitmap of current sink writer
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        // Just skip it.
-        Ok(())
-    }
 }
 
 pub struct IcebergSinkCommitter {
diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs
index f52eb0bfa36f8..4c7fc317edd3e 100644
--- a/src/connector/src/sink/kafka.rs
+++ b/src/connector/src/sink/kafka.rs
@@ -32,7 +32,8 @@ use serde_json::Value;
 use serde_with::{serde_as, DisplayFromStr};
 
 use super::{
-    Sink, SinkError, SINK_TYPE_APPEND_ONLY, SINK_TYPE_DEBEZIUM, SINK_TYPE_OPTION, SINK_TYPE_UPSERT,
+    Sink, SinkError, SinkParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_DEBEZIUM, SINK_TYPE_OPTION,
+    SINK_TYPE_UPSERT,
 };
 use crate::common::KafkaCommon;
 use crate::sink::utils::{
@@ -260,20 +261,19 @@ pub struct KafkaSink {
     schema: Schema,
     pk_indices: Vec<usize>,
     is_append_only: bool,
+    db_name: String,
+    sink_from_name: String,
 }
 
 impl KafkaSink {
-    pub fn new(
-        config: KafkaConfig,
-        schema: Schema,
-        pk_indices: Vec<usize>,
-        is_append_only: bool,
-    ) -> Self {
+    pub fn new(config: KafkaConfig, param: SinkParam) -> Self {
         Self {
             config,
-            schema,
-            pk_indices,
-            is_append_only,
+            schema: param.schema(),
+            pk_indices: param.pk_indices,
+            is_append_only: param.sink_type.is_append_only(),
+            db_name: param.db_name,
+            sink_from_name: param.sink_from_name,
         }
     }
 }
@@ -290,6 +290,8 @@ impl Sink for KafkaSink {
                 self.schema.clone(),
                 self.pk_indices.clone(),
                 self.is_append_only,
+                self.db_name.clone(),
+                self.sink_from_name.clone(),
                 format!("sink-{:?}", writer_param.executor_id),
             )
             .await?,
@@ -333,6 +335,8 @@ pub struct KafkaSinkWriter {
     schema: Schema,
     pk_indices: Vec<usize>,
     is_append_only: bool,
+    db_name: String,
+    sink_from_name: String,
 }
 
 impl KafkaSinkWriter {
@@ -341,6 +345,8 @@ impl KafkaSinkWriter {
         schema: Schema,
         pk_indices: Vec<usize>,
         is_append_only: bool,
+        db_name: String,
+        sink_from_name: String,
         identifier: String,
     ) -> Result<Self> {
         let inner: FutureProducer<PrivateLinkProducerContext> = {
@@ -376,6 +382,8 @@ impl KafkaSinkWriter {
             schema,
             pk_indices,
             is_append_only,
+            db_name,
+            sink_from_name,
         })
     }
 
@@ -430,10 +438,13 @@ impl KafkaSinkWriter {
                 Err((e, rec)) => {
                     err = e;
                     record = rec;
-                    // FIXME: Will there possibly exist other errors?
-                    assert!(err == KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull));
-                    tokio::time::sleep(self.config.retry_interval).await;
-                    continue;
+                    match err {
+                        KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull) => {
+                            tokio::time::sleep(self.config.retry_interval).await;
+                            continue;
+                        }
+                        _ => break,
+                    }
                 }
             }
         }
@@ -467,6 +478,8 @@ impl KafkaSinkWriter {
             chunk,
             ts_ms,
             DebeziumAdapterOpts::default(),
+            &self.db_name,
+            &self.sink_from_name,
         );
 
         #[for_await]
@@ -727,6 +740,8 @@ mod test {
             pk_indices,
             true,
             "test_sink_1".to_string(),
+            "test_db".into(),
+            "test_table".into(),
         )
         .await
         .unwrap();
@@ -815,8 +830,8 @@ mod test {
         ]);
 
         let json_chunk = chunk_to_json(chunk, &schema).unwrap();
-        let schema_json = schema_to_json(&schema);
-        assert_eq!(schema_json, serde_json::from_str::<Value>("{\"fields\":[{\"field\":\"before\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"after\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"source\",\"fields\":[{\"field\":\"db\",\"optional\":false,\"type\":\"string\"},{\"field\":\"table\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Source\",\"optional\":false,\"type\":\"struct\"},{\"field\":\"op\",\"optional\":false,\"type\":\"string\"},{\"field\":\"ts_ms\",\"optional\":false,\"type\":\"int64\"}],\"name\":\"RisingWave.RisingWave.RisingWave.Envelope\",\"optional\":false,\"type\":\"struct\"}").unwrap());
+        let schema_json = schema_to_json(&schema, "test_db", "test_table");
+        assert_eq!(schema_json, serde_json::from_str::<Value>("{\"fields\":[{\"field\":\"before\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.test_db.test_table.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"after\",\"fields\":[{\"field\":\"v1\",\"optional\":true,\"type\":\"int32\"},{\"field\":\"v2\",\"optional\":true,\"type\":\"float\"},{\"field\":\"v3\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.test_db.test_table.Key\",\"optional\":true,\"type\":\"struct\"},{\"field\":\"source\",\"fields\":[{\"field\":\"db\",\"optional\":false,\"type\":\"string\"},{\"field\":\"table\",\"optional\":true,\"type\":\"string\"}],\"name\":\"RisingWave.test_db.test_table.Source\",\"optional\":false,\"type\":\"struct\"},{\"field\":\"op\",\"optional\":false,\"type\":\"string\"},{\"field\":\"ts_ms\",\"optional\":false,\"type\":\"int64\"}],\"name\":\"RisingWave.test_db.test_table.Envelope\",\"optional\":false,\"type\":\"struct\"}").unwrap());
         assert_eq!(
             serde_json::from_str::<Value>(&json_chunk[0]).unwrap(),
             serde_json::from_str::<Value>("{\"v1\":0,\"v2\":0.0,\"v3\":{\"v4\":0,\"v5\":0.0}}")
diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs
index e303731e2afca..4d76a3235e381 100644
--- a/src/connector/src/sink/kinesis.rs
+++ b/src/connector/src/sink/kinesis.rs
@@ -22,7 +22,6 @@ use aws_sdk_kinesis::primitives::Blob;
 use aws_sdk_kinesis::Client as KinesisClient;
 use futures_async_stream::for_await;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_rpc_client::ConnectorClient;
 use serde_derive::Deserialize;
@@ -30,6 +29,7 @@ use serde_with::serde_as;
 use tokio_retry::strategy::{jitter, ExponentialBackoff};
 use tokio_retry::Retry;
 
+use super::SinkParam;
 use crate::common::KinesisCommon;
 use crate::sink::utils::{
     gen_append_only_message_stream, gen_debezium_message_stream, gen_upsert_message_stream,
@@ -48,20 +48,19 @@ pub struct KinesisSink {
     schema: Schema,
     pk_indices: Vec<usize>,
     is_append_only: bool,
+    db_name: String,
+    sink_from_name: String,
 }
 
 impl KinesisSink {
-    pub fn new(
-        config: KinesisSinkConfig,
-        schema: Schema,
-        pk_indices: Vec<usize>,
-        is_append_only: bool,
-    ) -> Self {
+    pub fn new(config: KinesisSinkConfig, param: SinkParam) -> Self {
         Self {
             config,
-            schema,
-            pk_indices,
-            is_append_only,
+            schema: param.schema(),
+            pk_indices: param.pk_indices,
+            is_append_only: param.sink_type.is_append_only(),
+            db_name: param.db_name,
+            sink_from_name: param.sink_from_name,
         }
     }
 }
@@ -100,6 +99,8 @@ impl Sink for KinesisSink {
             self.schema.clone(),
             self.pk_indices.clone(),
             self.is_append_only,
+            self.db_name.clone(),
+            self.sink_from_name.clone(),
         )
         .await
     }
@@ -142,6 +143,8 @@ pub struct KinesisSinkWriter {
     pk_indices: Vec<usize>,
     client: KinesisClient,
     is_append_only: bool,
+    db_name: String,
+    sink_from_name: String,
 }
 
 impl KinesisSinkWriter {
@@ -150,6 +153,8 @@ impl KinesisSinkWriter {
         schema: Schema,
         pk_indices: Vec<usize>,
         is_append_only: bool,
+        db_name: String,
+        sink_from_name: String,
     ) -> Result<Self> {
         let client = config
             .common
@@ -162,6 +167,8 @@ impl KinesisSinkWriter {
             pk_indices,
             client,
             is_append_only,
+            db_name,
+            sink_from_name,
         })
     }
 
@@ -225,6 +232,8 @@ impl KinesisSinkWriter {
             chunk,
             ts_ms,
             DebeziumAdapterOpts::default(),
+            &self.db_name,
+            &self.sink_from_name,
         );
 
         crate::impl_load_stream_write_record!(dbz_stream, self.put_record);
@@ -262,14 +271,6 @@ impl SinkWriter for KinesisSinkWriter {
     async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
         Ok(())
     }
-
-    async fn abort(&mut self) -> Result<()> {
-        Ok(())
-    }
-
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        Ok(())
-    }
 }
 
 #[macro_export]
diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs
index 814c91c6795cc..fd1de194671b3 100644
--- a/src/connector/src/sink/mod.rs
+++ b/src/connector/src/sink/mod.rs
@@ -19,6 +19,7 @@ pub mod coordinate;
 pub mod iceberg;
 pub mod kafka;
 pub mod kinesis;
+pub mod nats;
 pub mod redis;
 pub mod remote;
 #[cfg(any(test, madsim))]
@@ -26,6 +27,7 @@ pub mod test_sink;
 pub mod utils;
 
 use std::collections::HashMap;
+use std::sync::Arc;
 
 use ::clickhouse::error::Error as ClickHouseError;
 use anyhow::anyhow;
@@ -51,6 +53,7 @@ use crate::sink::clickhouse::CLICKHOUSE_SINK;
 use crate::sink::iceberg::{IcebergConfig, RemoteIcebergConfig, RemoteIcebergSink};
 use crate::sink::kafka::{KafkaConfig, KafkaSink, KAFKA_SINK};
 use crate::sink::kinesis::{KinesisSink, KinesisSinkConfig, KINESIS_SINK};
+use crate::sink::nats::{NatsConfig, NatsSink, NATS_SINK};
 use crate::sink::redis::{RedisConfig, RedisSink};
 use crate::sink::remote::{CoordinatedRemoteSink, RemoteConfig, RemoteSink};
 #[cfg(any(test, madsim))]
@@ -71,6 +74,8 @@ pub struct SinkParam {
     pub columns: Vec<ColumnDesc>,
     pub pk_indices: Vec<usize>,
     pub sink_type: SinkType,
+    pub db_name: String,
+    pub sink_from_name: String,
 }
 
 impl SinkParam {
@@ -88,6 +93,8 @@ impl SinkParam {
             sink_type: SinkType::from_proto(
                 PbSinkType::from_i32(pb_param.sink_type).expect("should be able to convert"),
             ),
+            db_name: pb_param.db_name,
+            sink_from_name: pb_param.sink_from_name,
         }
     }
 
@@ -100,6 +107,8 @@ impl SinkParam {
                 pk_indices: self.pk_indices.iter().map(|i| *i as u32).collect(),
             }),
             sink_type: self.sink_type.to_proto().into(),
+            db_name: self.db_name.clone(),
+            sink_from_name: self.sink_from_name.clone(),
         }
     }
 
@@ -122,6 +131,8 @@ impl From<SinkCatalog> for SinkParam {
             columns,
             pk_indices: sink_catalog.downstream_pk,
             sink_type: sink_catalog.sink_type,
+            db_name: sink_catalog.db_name,
+            sink_from_name: sink_catalog.sink_from_name,
         }
     }
 }
@@ -163,10 +174,14 @@ pub trait SinkWriter: Send + 'static {
     async fn barrier(&mut self, is_checkpoint: bool) -> Result<Self::CommitMetadata>;
 
     /// Clean up
-    async fn abort(&mut self) -> Result<()>;
+    async fn abort(&mut self) -> Result<()> {
+        Ok(())
+    }
 
     /// Update the vnode bitmap of current sink writer
-    async fn update_vnode_bitmap(&mut self, vnode_bitmap: Bitmap) -> Result<()>;
+    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc<Bitmap>) -> Result<()> {
+        Ok(())
+    }
 }
 
 #[async_trait]
@@ -230,10 +245,6 @@ impl<W: SinkWriterV1> SinkWriter for SinkWriterV1Adapter<W> {
     async fn abort(&mut self) -> Result<()> {
         self.inner.abort().await
     }
-
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        Ok(())
-    }
 }
 
 #[async_trait]
@@ -270,6 +281,7 @@ pub enum SinkConfig {
     RemoteIceberg(RemoteIcebergConfig),
     BlackHole,
     ClickHouse(Box<ClickHouseConfig>),
+    Nats(NatsConfig),
     #[cfg(any(test, madsim))]
     Test,
 }
@@ -303,17 +315,9 @@ impl SinkWriter for BlackHoleSink {
         Ok(())
     }
 
-    async fn abort(&mut self) -> Result<()> {
-        Ok(())
-    }
-
     async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
         Ok(())
     }
-
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        Ok(())
-    }
 }
 
 impl SinkConfig {
@@ -346,6 +350,7 @@ impl SinkConfig {
             ICEBERG_SINK => Ok(SinkConfig::Iceberg(IcebergConfig::from_hashmap(
                 properties,
             )?)),
+            NATS_SINK => Ok(SinkConfig::Nats(NatsConfig::from_hashmap(properties)?)),
             // Only in test or deterministic test, test sink is enabled.
             #[cfg(any(test, madsim))]
             TEST_SINK_NAME => Ok(SinkConfig::Test),
@@ -368,6 +373,7 @@ pub enum SinkImpl {
     Kinesis(KinesisSink),
     ClickHouse(ClickHouseSink),
     Iceberg(IcebergSink),
+    Nats(NatsSink),
     RemoteIceberg(RemoteIcebergSink),
     TestSink(BoxSink),
 }
@@ -382,6 +388,7 @@ impl SinkImpl {
             SinkImpl::Kinesis(_) => "kinesis",
             SinkImpl::ClickHouse(_) => "clickhouse",
             SinkImpl::Iceberg(_) => "iceberg",
+            SinkImpl::Nats(_) => "nats",
             SinkImpl::RemoteIceberg(_) => "iceberg",
             SinkImpl::TestSink(_) => "test",
         }
@@ -401,6 +408,7 @@ macro_rules! dispatch_sink {
             SinkImpl::Kinesis($sink) => $body,
             SinkImpl::ClickHouse($sink) => $body,
             SinkImpl::Iceberg($sink) => $body,
+            SinkImpl::Nats($sink) => $body,
             SinkImpl::RemoteIceberg($sink) => $body,
             SinkImpl::TestSink($sink) => $body,
         }
@@ -411,18 +419,8 @@ impl SinkImpl {
     pub fn new(cfg: SinkConfig, param: SinkParam) -> Result<Self> {
         Ok(match cfg {
             SinkConfig::Redis(cfg) => SinkImpl::Redis(RedisSink::new(cfg, param.schema())?),
-            SinkConfig::Kafka(cfg) => SinkImpl::Kafka(KafkaSink::new(
-                *cfg,
-                param.schema(),
-                param.pk_indices,
-                param.sink_type.is_append_only(),
-            )),
-            SinkConfig::Kinesis(cfg) => SinkImpl::Kinesis(KinesisSink::new(
-                *cfg,
-                param.schema(),
-                param.pk_indices,
-                param.sink_type.is_append_only(),
-            )),
+            SinkConfig::Kafka(cfg) => SinkImpl::Kafka(KafkaSink::new(*cfg, param)),
+            SinkConfig::Kinesis(cfg) => SinkImpl::Kinesis(KinesisSink::new(*cfg, param)),
             SinkConfig::Remote(cfg) => SinkImpl::Remote(RemoteSink::new(cfg, param)),
             SinkConfig::BlackHole => SinkImpl::BlackHole(BlackHoleSink),
             SinkConfig::ClickHouse(cfg) => SinkImpl::ClickHouse(ClickHouseSink::new(
@@ -432,6 +430,11 @@ impl SinkImpl {
                 param.sink_type.is_append_only(),
             )?),
             SinkConfig::Iceberg(cfg) => SinkImpl::Iceberg(IcebergSink::new(cfg, param)?),
+            SinkConfig::Nats(cfg) => SinkImpl::Nats(NatsSink::new(
+                cfg,
+                param.schema(),
+                param.sink_type.is_append_only(),
+            )),
             SinkConfig::RemoteIceberg(cfg) => {
                 SinkImpl::RemoteIceberg(CoordinatedRemoteSink(RemoteSink::new(cfg, param)))
             }
@@ -461,6 +464,8 @@ pub enum SinkError {
     Coordinator(anyhow::Error),
     #[error("ClickHouse error: {0}")]
     ClickHouse(String),
+    #[error("Nats error: {0}")]
+    Nats(anyhow::Error),
 }
 
 impl From<RpcError> for SinkError {
diff --git a/src/connector/src/sink/nats.rs b/src/connector/src/sink/nats.rs
new file mode 100644
index 0000000000000..c2408acbdab9d
--- /dev/null
+++ b/src/connector/src/sink/nats.rs
@@ -0,0 +1,157 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+use core::fmt::Debug;
+use std::collections::HashMap;
+
+use anyhow::anyhow;
+use async_nats::jetstream::context::Context;
+use risingwave_common::array::StreamChunk;
+use risingwave_common::catalog::Schema;
+use risingwave_common::error::anyhow_error;
+use risingwave_rpc_client::ConnectorClient;
+use serde_derive::Deserialize;
+use serde_with::serde_as;
+use tokio_retry::strategy::{jitter, ExponentialBackoff};
+use tokio_retry::Retry;
+
+use super::utils::chunk_to_json;
+use super::{DummySinkCommitCoordinator, SinkWriter, SinkWriterParam};
+use crate::common::NatsCommon;
+use crate::sink::{Result, Sink, SinkError, SINK_TYPE_APPEND_ONLY};
+
+pub const NATS_SINK: &str = "nats";
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+pub struct NatsConfig {
+    #[serde(flatten)]
+    pub common: NatsCommon,
+    // accept "append-only"
+    pub r#type: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct NatsSink {
+    pub config: NatsConfig,
+    schema: Schema,
+    is_append_only: bool,
+}
+
+// sink write
+pub struct NatsSinkWriter {
+    pub config: NatsConfig,
+    context: Context,
+    schema: Schema,
+}
+
+/// Basic data types for use with the nats interface
+impl NatsConfig {
+    pub fn from_hashmap(values: HashMap<String, String>) -> Result<Self> {
+        let config = serde_json::from_value::<NatsConfig>(serde_json::to_value(values).unwrap())
+            .map_err(|e| SinkError::Config(anyhow!(e)))?;
+        if config.r#type != SINK_TYPE_APPEND_ONLY {
+            Err(SinkError::Config(anyhow!(
+                "Nats sink only support append-only mode"
+            )))
+        } else {
+            Ok(config)
+        }
+    }
+}
+
+impl NatsSink {
+    pub fn new(config: NatsConfig, schema: Schema, is_append_only: bool) -> Self {
+        Self {
+            config,
+            schema,
+            is_append_only,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Sink for NatsSink {
+    type Coordinator = DummySinkCommitCoordinator;
+    type Writer = NatsSinkWriter;
+
+    async fn validate(&self, _client: Option<ConnectorClient>) -> Result<()> {
+        if !self.is_append_only {
+            return Err(SinkError::Nats(anyhow!(
+                "Nats sink only support append-only mode"
+            )));
+        }
+        match self.config.common.build_client().await {
+            Ok(_client) => {}
+            Err(error) => {
+                return Err(SinkError::Nats(anyhow_error!(
+                    "validate nats sink error: {:?}",
+                    error
+                )));
+            }
+        }
+        Ok(())
+    }
+
+    async fn new_writer(&self, _writer_env: SinkWriterParam) -> Result<Self::Writer> {
+        Ok(NatsSinkWriter::new(self.config.clone(), self.schema.clone()).await?)
+    }
+}
+
+impl NatsSinkWriter {
+    pub async fn new(config: NatsConfig, schema: Schema) -> Result<Self> {
+        let context = config
+            .common
+            .build_context()
+            .await
+            .map_err(|e| SinkError::Nats(anyhow_error!("nats sink error: {:?}", e)))?;
+        Ok::<_, SinkError>(Self {
+            config: config.clone(),
+            context,
+            schema: schema.clone(),
+        })
+    }
+
+    async fn append_only(&mut self, chunk: StreamChunk) -> Result<()> {
+        Retry::spawn(
+            ExponentialBackoff::from_millis(100).map(jitter).take(3),
+            || async {
+                let data = chunk_to_json(chunk.clone(), &self.schema).unwrap();
+                for item in data {
+                    self.context
+                        .publish(self.config.common.subject.clone(), item.into())
+                        .await
+                        .map_err(|e| SinkError::Nats(anyhow_error!("nats sink error: {:?}", e)))?;
+                }
+                Ok::<_, SinkError>(())
+            },
+        )
+        .await
+        .map_err(|e| SinkError::Nats(anyhow_error!("nats sink error: {:?}", e)))
+    }
+}
+
+#[async_trait::async_trait]
+impl SinkWriter for NatsSinkWriter {
+    async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> {
+        self.append_only(chunk).await
+    }
+
+    async fn begin_epoch(&mut self, _epoch_id: u64) -> Result<()> {
+        Ok(())
+    }
+
+    async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
+        Ok(())
+    }
+}
diff --git a/src/connector/src/sink/redis.rs b/src/connector/src/sink/redis.rs
index 1c059a1fcb16e..e5afa88c38b2f 100644
--- a/src/connector/src/sink/redis.rs
+++ b/src/connector/src/sink/redis.rs
@@ -14,7 +14,6 @@
 
 use async_trait::async_trait;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_rpc_client::ConnectorClient;
 
@@ -58,15 +57,7 @@ impl SinkWriter for RedisSinkWriter {
         todo!()
     }
 
-    async fn abort(&mut self) -> Result<()> {
-        todo!()
-    }
-
     async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
         todo!()
     }
-
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        todo!()
-    }
 }
diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs
index dcaaeb0389d95..403ee4c7b73e5 100644
--- a/src/connector/src/sink/remote.rs
+++ b/src/connector/src/sink/remote.rs
@@ -20,7 +20,6 @@ use async_trait::async_trait;
 use itertools::Itertools;
 use prost::Message;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::error::anyhow_error;
 use risingwave_common::types::DataType;
@@ -49,8 +48,13 @@ use crate::sink::{
 };
 use crate::ConnectorParams;
 
-pub const VALID_REMOTE_SINKS: [&str; 4] =
-    ["jdbc", REMOTE_ICEBERG_SINK, "deltalake", "elasticsearch-7"];
+pub const VALID_REMOTE_SINKS: [&str; 5] = [
+    "jdbc",
+    REMOTE_ICEBERG_SINK,
+    "deltalake",
+    "elasticsearch-7",
+    "cassandra",
+];
 
 pub fn is_valid_remote_sink(connector_type: &str) -> bool {
     VALID_REMOTE_SINKS.contains(&connector_type)
@@ -409,15 +413,6 @@ where
             Ok(<Self as HandleBarrierResponse>::non_checkpoint_return_value())
         }
     }
-
-    async fn abort(&mut self) -> Result<()> {
-        Ok(())
-    }
-
-    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Bitmap) -> Result<()> {
-        // TODO: handle scaling
-        Ok(())
-    }
 }
 
 pub struct RemoteCoordinator {
diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs
index 9b9adb3796629..1eb2f91e9af22 100644
--- a/src/connector/src/sink/utils.rs
+++ b/src/connector/src/sink/utils.rs
@@ -27,6 +27,8 @@ use tracing::warn;
 
 use crate::sink::{Result, SinkError};
 
+const DEBEZIUM_NAME_FIELD_PREFIX: &str = "RisingWave";
+
 pub struct DebeziumAdapterOpts {
     gen_tombstone: bool,
 }
@@ -39,6 +41,10 @@ impl Default for DebeziumAdapterOpts {
     }
 }
 
+fn concat_debezium_name_field(db_name: &str, sink_from_name: &str, value: &str) -> String {
+    DEBEZIUM_NAME_FIELD_PREFIX.to_owned() + "." + db_name + "." + sink_from_name + "." + value
+}
+
 #[try_stream(ok = (Option<Value>, Option<Value>), error = SinkError)]
 pub async fn gen_debezium_message_stream<'a>(
     schema: &'a Schema,
@@ -46,10 +52,12 @@ pub async fn gen_debezium_message_stream<'a>(
     chunk: StreamChunk,
     ts_ms: u64,
     opts: DebeziumAdapterOpts,
+    db_name: &'a str,
+    sink_from_name: &'a str,
 ) {
     let source_field = json!({
-        "db": "RisingWave",
-        "table": "RisingWave",
+        "db": db_name,
+        "table": sink_from_name,
     });
 
     let mut update_cache: Option<Map<String, Value>> = None;
@@ -60,13 +68,13 @@ pub async fn gen_debezium_message_stream<'a>(
                 "type": "struct",
                 "fields": fields_pk_to_json(&schema.fields, pk_indices),
                 "optional": false,
-                "name": "RisingWave.RisingWave.RisingWave.Key",
+                "name": concat_debezium_name_field(db_name, sink_from_name, "Key"),
             }),
             "payload": pk_to_json(row, &schema.fields, pk_indices)?,
         }));
         let event_object: Option<Value> = match op {
             Op::Insert => Some(json!({
-                "schema": schema_to_json(schema),
+                "schema": schema_to_json(schema, db_name, sink_from_name),
                 "payload": {
                     "before": null,
                     "after": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?,
@@ -77,7 +85,7 @@ pub async fn gen_debezium_message_stream<'a>(
             })),
             Op::Delete => {
                 let value_obj = Some(json!({
-                    "schema": schema_to_json(schema),
+                    "schema": schema_to_json(schema, db_name, sink_from_name),
                     "payload": {
                         "before": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?,
                         "after": null,
@@ -107,7 +115,7 @@ pub async fn gen_debezium_message_stream<'a>(
             Op::UpdateInsert => {
                 if let Some(before) = update_cache.take() {
                     Some(json!({
-                        "schema": schema_to_json(schema),
+                        "schema": schema_to_json(schema, db_name, sink_from_name),
                         "payload": {
                             "before": before,
                             "after": record_to_json(row, &schema.fields, TimestampHandlingMode::Milli)?,
@@ -129,27 +137,27 @@ pub async fn gen_debezium_message_stream<'a>(
     }
 }
 
-pub(crate) fn schema_to_json(schema: &Schema) -> Value {
+pub(crate) fn schema_to_json(schema: &Schema, db_name: &str, sink_from_name: &str) -> Value {
     let mut schema_fields = Vec::new();
     schema_fields.push(json!({
         "type": "struct",
         "fields": fields_to_json(&schema.fields),
         "optional": true,
         "field": "before",
-        "name": "RisingWave.RisingWave.RisingWave.Key",
+        "name": concat_debezium_name_field(db_name, sink_from_name, "Key"),
     }));
     schema_fields.push(json!({
         "type": "struct",
         "fields": fields_to_json(&schema.fields),
         "optional": true,
         "field": "after",
-        "name": "RisingWave.RisingWave.RisingWave.Key",
+        "name": concat_debezium_name_field(db_name, sink_from_name, "Key"),
     }));
 
     schema_fields.push(json!({
         "type": "struct",
         "optional": false,
-        "name": "RisingWave.RisingWave.RisingWave.Source",
+        "name": concat_debezium_name_field(db_name, sink_from_name, "Source"),
         "fields": vec![
             json!({
                 "type": "string",
@@ -178,7 +186,7 @@ pub(crate) fn schema_to_json(schema: &Schema) -> Value {
         "type": "struct",
         "fields": schema_fields,
         "optional": false,
-        "name": "RisingWave.RisingWave.RisingWave.Envelope",
+        "name": concat_debezium_name_field(db_name, sink_from_name, "Envelope"),
     })
 }
 
diff --git a/src/connector/src/source/base.rs b/src/connector/src/source/base.rs
index 8b5b3287fb445..5a94b36a6b8e8 100644
--- a/src/connector/src/source/base.rs
+++ b/src/connector/src/source/base.rs
@@ -25,28 +25,31 @@ use itertools::Itertools;
 use parking_lot::Mutex;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::catalog::TableId;
-use risingwave_common::error::{ErrorCode, ErrorSuppressor, Result as RwResult, RwError};
+use risingwave_common::error::{ErrorSuppressor, RwError};
 use risingwave_common::types::{JsonbVal, Scalar};
-use risingwave_pb::connector_service::PbTableSchema;
 use risingwave_pb::source::ConnectorSplit;
 use risingwave_rpc_client::ConnectorClient;
-use serde::{Deserialize, Serialize};
 
 use super::datagen::DatagenMeta;
-use super::filesystem::{FsSplit, S3FileReader, S3Properties, S3SplitEnumerator, S3_CONNECTOR};
+use super::filesystem::{FsSplit, S3Properties, S3_CONNECTOR};
 use super::google_pubsub::GooglePubsubMeta;
 use super::kafka::KafkaMeta;
 use super::monitor::SourceMetrics;
+use super::nats::enumerator::NatsSplitEnumerator;
+use super::nats::source::NatsSplitReader;
 use super::nexmark::source::message::NexmarkMeta;
 use crate::parser::ParserConfig;
 use crate::source::cdc::{
-    CdcProperties, CdcSplitReader, DebeziumCdcSplit, DebeziumSplitEnumerator, CITUS_CDC_CONNECTOR,
-    MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR,
+    CdcProperties, CdcSplitReader, Citus, CitusDebeziumSplitEnumerator, DebeziumCdcSplit,
+    DebeziumSplitEnumerator, Mysql, MysqlDebeziumSplitEnumerator, Postgres,
+    PostgresDebeziumSplitEnumerator, CITUS_CDC_CONNECTOR, MYSQL_CDC_CONNECTOR,
+    POSTGRES_CDC_CONNECTOR,
 };
 use crate::source::datagen::{
     DatagenProperties, DatagenSplit, DatagenSplitEnumerator, DatagenSplitReader, DATAGEN_CONNECTOR,
 };
 use crate::source::dummy_connector::DummySplitReader;
+use crate::source::filesystem::{S3FileReader, S3SplitEnumerator};
 use crate::source::google_pubsub::{
     PubsubProperties, PubsubSplit, PubsubSplitEnumerator, PubsubSplitReader,
     GOOGLE_PUBSUB_CONNECTOR,
@@ -59,6 +62,8 @@ use crate::source::kinesis::source::reader::KinesisSplitReader;
 use crate::source::kinesis::split::KinesisSplit;
 use crate::source::kinesis::{KinesisProperties, KINESIS_CONNECTOR};
 use crate::source::monitor::EnumeratorMetrics;
+use crate::source::nats::split::NatsSplit;
+use crate::source::nats::{NatsProperties, NATS_CONNECTOR};
 use crate::source::nexmark::source::reader::NexmarkSplitReader;
 use crate::source::nexmark::{
     NexmarkProperties, NexmarkSplit, NexmarkSplitEnumerator, NEXMARK_CONNECTOR,
@@ -168,10 +173,9 @@ impl SourceContext {
         ctx
     }
 
-    pub(crate) fn report_user_source_error(&self, e: RwError) -> RwResult<()> {
-        // Repropagate the error if batch
+    pub(crate) fn report_user_source_error(&self, e: RwError) {
         if self.source_info.fragment_id == u32::MAX {
-            return Err(e);
+            return;
         }
         let mut err_str = e.inner().to_string();
         if let Some(suppressor) = &self.error_suppressor
@@ -194,7 +198,6 @@ impl SourceContext {
                 &self.source_info.source_id.table_id.to_string(),
             ])
             .inc();
-        Ok(())
     }
 }
 
@@ -287,7 +290,7 @@ pub trait SplitReader: Sized {
     fn into_stream(self) -> BoxSourceWithStateStream;
 }
 
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Clone, Debug)]
 pub enum ConnectorProperties {
     Kafka(Box<KafkaProperties>),
     Pulsar(Box<PulsarProperties>),
@@ -295,64 +298,21 @@ pub enum ConnectorProperties {
     Nexmark(Box<NexmarkProperties>),
     Datagen(Box<DatagenProperties>),
     S3(Box<S3Properties>),
-    MySqlCdc(Box<CdcProperties>),
-    PostgresCdc(Box<CdcProperties>),
-    CitusCdc(Box<CdcProperties>),
+    MysqlCdc(Box<CdcProperties<Mysql>>),
+    PostgresCdc(Box<CdcProperties<Postgres>>),
+    CitusCdc(Box<CdcProperties<Citus>>),
     GooglePubsub(Box<PubsubProperties>),
+    Nats(Box<NatsProperties>),
     Dummy(Box<()>),
 }
 
 impl ConnectorProperties {
-    fn new_cdc_properties(
-        connector_name: &str,
-        properties: HashMap<String, String>,
-    ) -> Result<Self> {
-        match connector_name {
-            MYSQL_CDC_CONNECTOR => Ok(Self::MySqlCdc(Box::new(CdcProperties {
-                props: properties,
-                source_type: "mysql".to_string(),
-                ..Default::default()
-            }))),
-            POSTGRES_CDC_CONNECTOR => Ok(Self::PostgresCdc(Box::new(CdcProperties {
-                props: properties,
-                source_type: "postgres".to_string(),
-                ..Default::default()
-            }))),
-            CITUS_CDC_CONNECTOR => Ok(Self::CitusCdc(Box::new(CdcProperties {
-                props: properties,
-                source_type: "citus".to_string(),
-                ..Default::default()
-            }))),
-            _ => Err(anyhow!("unexpected cdc connector '{}'", connector_name,)),
-        }
-    }
-
-    pub fn init_cdc_properties(&mut self, table_schema: Option<PbTableSchema>) {
-        match self {
-            ConnectorProperties::MySqlCdc(c)
-            | ConnectorProperties::PostgresCdc(c)
-            | ConnectorProperties::CitusCdc(c) => {
-                c.table_schema = table_schema;
-            }
-            _ => {}
-        }
-    }
-
-    pub fn is_cdc_connector(&self) -> bool {
-        matches!(
-            self,
-            ConnectorProperties::MySqlCdc(_)
-                | ConnectorProperties::PostgresCdc(_)
-                | ConnectorProperties::CitusCdc(_)
-        )
-    }
-
     pub fn support_multiple_splits(&self) -> bool {
         matches!(self, ConnectorProperties::Kafka(_))
     }
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, EnumAsInner, PartialEq, Hash)]
+#[derive(Debug, Clone, EnumAsInner, PartialEq, Hash)]
 pub enum SplitImpl {
     Kafka(KafkaSplit),
     Pulsar(PulsarSplit),
@@ -360,9 +320,10 @@ pub enum SplitImpl {
     Nexmark(NexmarkSplit),
     Datagen(DatagenSplit),
     GooglePubsub(PubsubSplit),
-    MySqlCdc(DebeziumCdcSplit),
-    PostgresCdc(DebeziumCdcSplit),
-    CitusCdc(DebeziumCdcSplit),
+    MysqlCdc(DebeziumCdcSplit<Mysql>),
+    PostgresCdc(DebeziumCdcSplit<Postgres>),
+    CitusCdc(DebeziumCdcSplit<Citus>),
+    Nats(NatsSplit),
     S3(FsSplit),
 }
 
@@ -392,10 +353,11 @@ pub enum SplitReaderImpl {
     Nexmark(Box<NexmarkSplitReader>),
     Pulsar(Box<PulsarSplitReader>),
     Datagen(Box<DatagenSplitReader>),
-    MySqlCdc(Box<CdcSplitReader>),
-    PostgresCdc(Box<CdcSplitReader>),
-    CitusCdc(Box<CdcSplitReader>),
+    MysqlCdc(Box<CdcSplitReader<Mysql>>),
+    PostgresCdc(Box<CdcSplitReader<Postgres>>),
+    CitusCdc(Box<CdcSplitReader<Citus>>),
     GooglePubsub(Box<PubsubSplitReader>),
+    Nats(Box<NatsSplitReader>),
 }
 
 pub enum SplitEnumeratorImpl {
@@ -404,11 +366,12 @@ pub enum SplitEnumeratorImpl {
     Kinesis(KinesisSplitEnumerator),
     Nexmark(NexmarkSplitEnumerator),
     Datagen(DatagenSplitEnumerator),
-    MySqlCdc(DebeziumSplitEnumerator),
-    PostgresCdc(DebeziumSplitEnumerator),
-    CitusCdc(DebeziumSplitEnumerator),
+    MysqlCdc(MysqlDebeziumSplitEnumerator),
+    PostgresCdc(PostgresDebeziumSplitEnumerator),
+    CitusCdc(CitusDebeziumSplitEnumerator),
     GooglePubsub(PubsubSplitEnumerator),
     S3(S3SplitEnumerator),
+    Nats(NatsSplitEnumerator),
 }
 
 impl_connector_properties! {
@@ -418,10 +381,8 @@ impl_connector_properties! {
     { Nexmark, NEXMARK_CONNECTOR },
     { Datagen, DATAGEN_CONNECTOR },
     { S3, S3_CONNECTOR },
-    { MySqlCdc, MYSQL_CDC_CONNECTOR },
-    { PostgresCdc, POSTGRES_CDC_CONNECTOR },
-    { CitusCdc, CITUS_CDC_CONNECTOR },
-    { GooglePubsub, GOOGLE_PUBSUB_CONNECTOR}
+    { GooglePubsub, GOOGLE_PUBSUB_CONNECTOR},
+    { Nats, NATS_CONNECTOR }
 }
 
 impl_split_enumerator! {
@@ -430,11 +391,12 @@ impl_split_enumerator! {
     { Kinesis, KinesisSplitEnumerator },
     { Nexmark, NexmarkSplitEnumerator },
     { Datagen, DatagenSplitEnumerator },
-    { MySqlCdc, DebeziumSplitEnumerator },
+    { MysqlCdc, DebeziumSplitEnumerator },
     { PostgresCdc, DebeziumSplitEnumerator },
     { CitusCdc, DebeziumSplitEnumerator },
     { GooglePubsub, PubsubSplitEnumerator},
-    { S3, S3SplitEnumerator }
+    { S3, S3SplitEnumerator },
+    { Nats, NatsSplitEnumerator }
 }
 
 impl_split! {
@@ -444,10 +406,11 @@ impl_split! {
     { Nexmark, NEXMARK_CONNECTOR, NexmarkSplit },
     { Datagen, DATAGEN_CONNECTOR, DatagenSplit },
     { GooglePubsub, GOOGLE_PUBSUB_CONNECTOR, PubsubSplit },
-    { MySqlCdc, MYSQL_CDC_CONNECTOR, DebeziumCdcSplit },
-    { PostgresCdc, POSTGRES_CDC_CONNECTOR, DebeziumCdcSplit },
-    { CitusCdc, CITUS_CDC_CONNECTOR, DebeziumCdcSplit },
-    { S3, S3_CONNECTOR, FsSplit }
+    { MysqlCdc, MYSQL_CDC_CONNECTOR, DebeziumCdcSplit<Mysql> },
+    { PostgresCdc, POSTGRES_CDC_CONNECTOR, DebeziumCdcSplit<Postgres> },
+    { CitusCdc, CITUS_CDC_CONNECTOR, DebeziumCdcSplit<Citus> },
+    { S3, S3_CONNECTOR, FsSplit },
+    { Nats, NATS_CONNECTOR, NatsSplit }
 }
 
 impl_split_reader! {
@@ -457,10 +420,11 @@ impl_split_reader! {
     { Kinesis, KinesisSplitReader },
     { Nexmark, NexmarkSplitReader },
     { Datagen, DatagenSplitReader },
-    { MySqlCdc, CdcSplitReader},
+    { MysqlCdc, CdcSplitReader},
     { PostgresCdc, CdcSplitReader},
     { CitusCdc, CdcSplitReader },
     { GooglePubsub, PubsubSplitReader },
+    { Nats, NatsSplitReader },
     { Dummy, DummySplitReader }
 }
 
@@ -484,7 +448,6 @@ pub struct SourceMessage {
     pub payload: Option<Vec<u8>>,
     pub offset: String,
     pub split_id: SplitId,
-
     pub meta: SourceMeta,
 }
 
@@ -556,7 +519,7 @@ mod tests {
         let offset_str = "{\"sourcePartition\":{\"server\":\"RW_CDC_mydb.products\"},\"sourceOffset\":{\"transaction_id\":null,\"ts_sec\":1670407377,\"file\":\"binlog.000001\",\"pos\":98587,\"row\":2,\"server_id\":1,\"event\":2}}";
         let mysql_split = MySqlCdcSplit::new(1001, offset_str.to_string());
         let split = DebeziumCdcSplit::new(Some(mysql_split), None);
-        let split_impl = SplitImpl::MySqlCdc(split);
+        let split_impl = SplitImpl::MysqlCdc(split);
         let encoded_split = split_impl.encode_to_bytes();
         let restored_split_impl = SplitImpl::restore_from_bytes(encoded_split.as_ref())?;
         assert_eq!(
@@ -644,8 +607,7 @@ mod tests {
         ));
 
         let conn_props = ConnectorProperties::extract(user_props_mysql).unwrap();
-        if let ConnectorProperties::MySqlCdc(c) = conn_props {
-            assert_eq!(c.source_type, "mysql");
+        if let ConnectorProperties::MysqlCdc(c) = conn_props {
             assert_eq!(c.props.get("connector_node_addr").unwrap(), "localhost");
             assert_eq!(c.props.get("database.hostname").unwrap(), "127.0.0.1");
             assert_eq!(c.props.get("database.port").unwrap(), "3306");
@@ -659,7 +621,6 @@ mod tests {
 
         let conn_props = ConnectorProperties::extract(user_props_postgres).unwrap();
         if let ConnectorProperties::PostgresCdc(c) = conn_props {
-            assert_eq!(c.source_type, "postgres");
             assert_eq!(c.props.get("connector_node_addr").unwrap(), "localhost");
             assert_eq!(c.props.get("database.hostname").unwrap(), "127.0.0.1");
             assert_eq!(c.props.get("database.port").unwrap(), "5432");
diff --git a/src/connector/src/source/cdc/enumerator/mod.rs b/src/connector/src/source/cdc/enumerator/mod.rs
index 91d2bf0bb7bbf..1c689026e568b 100644
--- a/src/connector/src/source/cdc/enumerator/mod.rs
+++ b/src/connector/src/source/cdc/enumerator/mod.rs
@@ -12,38 +12,43 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::marker::PhantomData;
 use std::str::FromStr;
 
 use anyhow::anyhow;
 use async_trait::async_trait;
 use itertools::Itertools;
 use risingwave_common::util::addr::HostAddr;
-use risingwave_pb::connector_service::SourceType as PbSourceType;
+use risingwave_pb::connector_service::SourceType;
 
 use crate::source::cdc::{
-    CdcProperties, CdcSplitBase, DebeziumCdcSplit, MySqlCdcSplit, PostgresCdcSplit,
+    CdcProperties, CdcSourceTypeTrait, CdcSplitBase, Citus, DebeziumCdcSplit, MySqlCdcSplit, Mysql,
+    Postgres, PostgresCdcSplit,
 };
 use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
 
 pub const DATABASE_SERVERS_KEY: &str = "database.servers";
 
 #[derive(Debug)]
-pub struct DebeziumSplitEnumerator {
+pub struct DebeziumSplitEnumerator<T: CdcSourceTypeTrait> {
     /// The source_id in the catalog
     source_id: u32,
-    source_type: PbSourceType,
     worker_node_addrs: Vec<HostAddr>,
+    _phantom: PhantomData<T>,
 }
 
 #[async_trait]
-impl SplitEnumerator for DebeziumSplitEnumerator {
-    type Properties = CdcProperties;
-    type Split = DebeziumCdcSplit;
+impl<T: CdcSourceTypeTrait> SplitEnumerator for DebeziumSplitEnumerator<T>
+where
+    Self: ListCdcSplits<CdcSourceType = T>,
+{
+    type Properties = CdcProperties<T>;
+    type Split = DebeziumCdcSplit<T>;
 
     async fn new(
-        props: CdcProperties,
+        props: CdcProperties<T>,
         context: SourceEnumeratorContextRef,
-    ) -> anyhow::Result<DebeziumSplitEnumerator> {
+    ) -> anyhow::Result<Self> {
         let connector_client = context.connector_client.clone().ok_or_else(|| {
             anyhow!("connector node endpoint not specified or unable to connect to connector node")
         })?;
@@ -59,68 +64,91 @@ impl SplitEnumerator for DebeziumSplitEnumerator {
             .transpose()?
             .unwrap_or_default();
 
-        let source_type = props.get_source_type_pb()?;
+        assert_eq!(
+            props.get_source_type_pb(),
+            SourceType::from(T::source_type())
+        );
+
         // validate connector properties
         connector_client
             .validate_source_properties(
                 context.info.source_id as u64,
-                props.get_source_type_pb()?,
+                props.get_source_type_pb(),
                 props.props,
-                props.table_schema,
+                Some(props.table_schema),
             )
             .await?;
 
         tracing::debug!("validate cdc source properties success");
         Ok(Self {
             source_id: context.info.source_id,
-            source_type,
             worker_node_addrs: server_addrs,
+            _phantom: PhantomData,
         })
     }
 
-    async fn list_splits(&mut self) -> anyhow::Result<Vec<DebeziumCdcSplit>> {
-        match self.source_type {
-            PbSourceType::Mysql => {
-                // CDC source only supports single split
-                let split = MySqlCdcSplit {
-                    inner: CdcSplitBase::new(self.source_id, None),
-                };
-                let dbz_split = DebeziumCdcSplit {
-                    mysql_split: Some(split),
-                    pg_split: None,
-                };
-                Ok(vec![dbz_split])
-            }
-            PbSourceType::Postgres => {
+    async fn list_splits(&mut self) -> anyhow::Result<Vec<DebeziumCdcSplit<T>>> {
+        Ok(self.list_cdc_splits())
+    }
+}
+
+pub trait ListCdcSplits {
+    type CdcSourceType: CdcSourceTypeTrait;
+    fn list_cdc_splits(&mut self) -> Vec<DebeziumCdcSplit<Self::CdcSourceType>>;
+}
+
+impl ListCdcSplits for DebeziumSplitEnumerator<Mysql> {
+    type CdcSourceType = Mysql;
+
+    fn list_cdc_splits(&mut self) -> Vec<DebeziumCdcSplit<Self::CdcSourceType>> {
+        // CDC source only supports single split
+        let split = MySqlCdcSplit {
+            inner: CdcSplitBase::new(self.source_id, None),
+        };
+        let dbz_split = DebeziumCdcSplit {
+            mysql_split: Some(split),
+            pg_split: None,
+            _phantom: PhantomData,
+        };
+        vec![dbz_split]
+    }
+}
+
+impl ListCdcSplits for DebeziumSplitEnumerator<Postgres> {
+    type CdcSourceType = Postgres;
+
+    fn list_cdc_splits(&mut self) -> Vec<DebeziumCdcSplit<Self::CdcSourceType>> {
+        let split = PostgresCdcSplit {
+            inner: CdcSplitBase::new(self.source_id, None),
+            server_addr: None,
+        };
+        let dbz_split = DebeziumCdcSplit {
+            mysql_split: None,
+            pg_split: Some(split),
+            _phantom: Default::default(),
+        };
+        vec![dbz_split]
+    }
+}
+
+impl ListCdcSplits for DebeziumSplitEnumerator<Citus> {
+    type CdcSourceType = Citus;
+
+    fn list_cdc_splits(&mut self) -> Vec<DebeziumCdcSplit<Self::CdcSourceType>> {
+        self.worker_node_addrs
+            .iter()
+            .enumerate()
+            .map(|(id, addr)| {
                 let split = PostgresCdcSplit {
-                    inner: CdcSplitBase::new(self.source_id, None),
-                    server_addr: None,
+                    inner: CdcSplitBase::new(id as u32, None),
+                    server_addr: Some(addr.to_string()),
                 };
-                let dbz_split = DebeziumCdcSplit {
+                DebeziumCdcSplit {
                     mysql_split: None,
                     pg_split: Some(split),
-                };
-                Ok(vec![dbz_split])
-            }
-            PbSourceType::Citus => {
-                let splits = self
-                    .worker_node_addrs
-                    .iter()
-                    .enumerate()
-                    .map(|(id, addr)| {
-                        let split = PostgresCdcSplit {
-                            inner: CdcSplitBase::new(id as u32, None),
-                            server_addr: Some(addr.to_string()),
-                        };
-                        DebeziumCdcSplit {
-                            mysql_split: None,
-                            pg_split: Some(split),
-                        }
-                    })
-                    .collect_vec();
-                Ok(splits)
-            }
-            _ => Err(anyhow!("unexpected source type")),
-        }
+                    _phantom: Default::default(),
+                }
+            })
+            .collect_vec()
     }
 }
diff --git a/src/connector/src/source/cdc/mod.rs b/src/connector/src/source/cdc/mod.rs
index 308e6ed88206c..86a8b16adec02 100644
--- a/src/connector/src/source/cdc/mod.rs
+++ b/src/connector/src/source/cdc/mod.rs
@@ -15,34 +15,57 @@
 pub mod enumerator;
 pub mod source;
 pub mod split;
-
 use std::collections::HashMap;
+use std::marker::PhantomData;
 
-use anyhow::anyhow;
 pub use enumerator::*;
-use risingwave_pb::connector_service::{SourceType, TableSchema};
-use serde::Deserialize;
+use paste::paste;
+use risingwave_common::catalog::{ColumnDesc, Field, Schema};
+use risingwave_pb::connector_service::{PbSourceType, PbTableSchema, SourceType, TableSchema};
 pub use source::*;
 pub use split::*;
 
-pub const MYSQL_CDC_CONNECTOR: &str = "mysql-cdc";
-pub const POSTGRES_CDC_CONNECTOR: &str = "postgres-cdc";
-pub const CITUS_CDC_CONNECTOR: &str = "citus-cdc";
+use crate::impl_cdc_source_type;
+use crate::source::ConnectorProperties;
+
+pub const CDC_CONNECTOR_NAME_SUFFIX: &str = "-cdc";
+
+pub const MYSQL_CDC_CONNECTOR: &str = Mysql::CDC_CONNECTOR_NAME;
+pub const POSTGRES_CDC_CONNECTOR: &str = Postgres::CDC_CONNECTOR_NAME;
+pub const CITUS_CDC_CONNECTOR: &str = Citus::CDC_CONNECTOR_NAME;
+
+pub trait CdcSourceTypeTrait: Send + Sync + Clone + 'static {
+    const CDC_CONNECTOR_NAME: &'static str;
+    fn source_type() -> CdcSourceType;
+}
 
-#[derive(Clone, Debug, Deserialize, Default)]
-pub struct CdcProperties {
-    /// Type of the cdc source, e.g. mysql, postgres
-    pub source_type: String,
+impl_cdc_source_type!({ Mysql, "mysql" }, { Postgres, "postgres" }, { Citus, "citus" });
+
+#[derive(Clone, Debug, Default)]
+pub struct CdcProperties<T: CdcSourceTypeTrait> {
     /// Properties specified in the WITH clause by user
     pub props: HashMap<String, String>,
 
     /// Schema of the source specified by users
-    pub table_schema: Option<TableSchema>,
+    pub table_schema: TableSchema,
+
+    pub _phantom: PhantomData<T>,
 }
 
-impl CdcProperties {
-    pub fn get_source_type_pb(&self) -> anyhow::Result<SourceType> {
-        SourceType::from_str_name(&self.source_type.to_ascii_uppercase())
-            .ok_or_else(|| anyhow!("unknown source type: {}", self.source_type))
+impl<T: CdcSourceTypeTrait> CdcProperties<T> {
+    pub fn get_source_type_pb(&self) -> SourceType {
+        SourceType::from(T::source_type())
+    }
+
+    pub fn schema(&self) -> Schema {
+        Schema {
+            fields: self
+                .table_schema
+                .columns
+                .iter()
+                .map(ColumnDesc::from)
+                .map(Field::from)
+                .collect(),
+        }
     }
 }
diff --git a/src/connector/src/source/cdc/source/reader.rs b/src/connector/src/source/cdc/source/reader.rs
index 200c91a8a5051..974ec8877d2f6 100644
--- a/src/connector/src/source/cdc/source/reader.rs
+++ b/src/connector/src/source/cdc/source/reader.rs
@@ -16,28 +16,26 @@ use std::str::FromStr;
 
 use anyhow::{anyhow, Result};
 use async_trait::async_trait;
-use futures::{pin_mut, StreamExt, TryStreamExt};
+use futures::pin_mut;
 use futures_async_stream::try_stream;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_pb::connector_service::GetEventStreamResponse;
 
-use crate::impl_common_split_reader_logic;
 use crate::parser::ParserConfig;
 use crate::source::base::SourceMessage;
-use crate::source::cdc::CdcProperties;
+use crate::source::cdc::{CdcProperties, CdcSourceType, CdcSourceTypeTrait, DebeziumCdcSplit};
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
 use crate::source::{
     BoxSourceWithStateStream, Column, SourceContextRef, SplitId, SplitImpl, SplitMetaData,
     SplitReader,
 };
 
-impl_common_split_reader_logic!(CdcSplitReader, CdcProperties);
-
-pub struct CdcSplitReader {
+pub struct CdcSplitReader<T: CdcSourceTypeTrait> {
     source_id: u64,
     start_offset: Option<String>,
     // host address of worker node for a Citus cluster
     server_addr: Option<String>,
-    conn_props: CdcProperties,
+    conn_props: CdcProperties<T>,
 
     split_id: SplitId,
     // whether the full snapshot phase is done
@@ -47,22 +45,25 @@ pub struct CdcSplitReader {
 }
 
 #[async_trait]
-impl SplitReader for CdcSplitReader {
-    type Properties = CdcProperties;
+impl<T: CdcSourceTypeTrait> SplitReader for CdcSplitReader<T>
+where
+    DebeziumCdcSplit<T>: TryFrom<SplitImpl, Error = anyhow::Error>,
+{
+    type Properties = CdcProperties<T>;
 
     #[allow(clippy::unused_async)]
     async fn new(
-        conn_props: CdcProperties,
+        conn_props: CdcProperties<T>,
         splits: Vec<SplitImpl>,
         parser_config: ParserConfig,
         source_ctx: SourceContextRef,
         _columns: Option<Vec<Column>>,
     ) -> Result<Self> {
         assert_eq!(splits.len(), 1);
-        let split = splits.into_iter().next().unwrap();
+        let split = DebeziumCdcSplit::<T>::try_from(splits.into_iter().next().unwrap())?;
         let split_id = split.id();
-        match split {
-            SplitImpl::MySqlCdc(split) | SplitImpl::PostgresCdc(split) => Ok(Self {
+        match T::source_type() {
+            CdcSourceType::Mysql | CdcSourceType::Postgres => Ok(Self {
                 source_id: split.split_id() as u64,
                 start_offset: split.start_offset().clone(),
                 server_addr: None,
@@ -72,7 +73,7 @@ impl SplitReader for CdcSplitReader {
                 parser_config,
                 source_ctx,
             }),
-            SplitImpl::CitusCdc(split) => Ok(Self {
+            CdcSourceType::Citus => Ok(Self {
                 source_id: split.split_id() as u64,
                 start_offset: split.start_offset().clone(),
                 server_addr: split.server_addr().clone(),
@@ -82,20 +83,21 @@ impl SplitReader for CdcSplitReader {
                 parser_config,
                 source_ctx,
             }),
-
-            _ => Err(anyhow!(
-                "failed to create cdc split reader: invalid splis info"
-            )),
         }
     }
 
     fn into_stream(self) -> BoxSourceWithStateStream {
-        self.into_chunk_stream()
+        let parser_config = self.parser_config.clone();
+        let source_context = self.source_ctx.clone();
+        into_chunk_stream(self, parser_config, source_context)
     }
 }
 
-impl CdcSplitReader {
-    #[try_stream(boxed, ok = Vec<SourceMessage>, error = anyhow::Error)]
+impl<T: CdcSourceTypeTrait> CommonSplitReader for CdcSplitReader<T>
+where
+    Self: SplitReader,
+{
+    #[try_stream(ok = Vec<SourceMessage>, error = anyhow::Error)]
     async fn into_data_stream(self) {
         let cdc_client = self.source_ctx.connector_client.clone().ok_or_else(|| {
             anyhow!("connector node endpoint not specified or unable to connect to connector node")
@@ -122,7 +124,7 @@ impl CdcSplitReader {
         let cdc_stream = cdc_client
             .start_source_stream(
                 self.source_id,
-                self.conn_props.get_source_type_pb()?,
+                self.conn_props.get_source_type_pb(),
                 self.start_offset,
                 properties,
                 self.snapshot_done,
diff --git a/src/connector/src/source/cdc/split.rs b/src/connector/src/source/cdc/split.rs
index e72fe97b5658f..9d13a3c5a6eac 100644
--- a/src/connector/src/source/cdc/split.rs
+++ b/src/connector/src/source/cdc/split.rs
@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::marker::PhantomData;
 
 use anyhow::anyhow;
 use risingwave_common::types::JsonbVal;
 use serde::{Deserialize, Serialize};
 
+use crate::source::cdc::CdcSourceTypeTrait;
+use crate::source::external::DebeziumOffset;
 use crate::source::{SplitId, SplitMetaData};
 
 /// The base states of a CDC split, which will be persisted to checkpoint.
@@ -39,37 +41,6 @@ impl CdcSplitBase {
     }
 }
 
-// Example debezium offset for Postgres:
-// {
-//     "sourcePartition":
-//     {
-//         "server": "RW_CDC_public.te"
-//     },
-//     "sourceOffset":
-//     {
-//         "last_snapshot_record": false,
-//         "lsn": 29973552,
-//         "txId": 1046,
-//         "ts_usec": 1670826189008456,
-//         "snapshot": true
-//     }
-// }
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct DebeziumOffset {
-    #[serde(rename = "sourcePartition")]
-    source_partition: HashMap<String, String>,
-    #[serde(rename = "sourceOffset")]
-    source_offset: DebeziumSourceOffset,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct DebeziumSourceOffset {
-    // postgres snapshot progress
-    last_snapshot_record: Option<bool>,
-    // mysql snapshot progress
-    snapshot: Option<bool>,
-}
-
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
 pub struct MySqlCdcSplit {
     pub inner: CdcSplitBase,
@@ -151,12 +122,15 @@ impl PostgresCdcSplit {
 }
 
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
-pub struct DebeziumCdcSplit {
+pub struct DebeziumCdcSplit<T: CdcSourceTypeTrait> {
     pub mysql_split: Option<MySqlCdcSplit>,
     pub pg_split: Option<PostgresCdcSplit>,
+
+    #[serde(skip)]
+    pub _phantom: PhantomData<T>,
 }
 
-impl SplitMetaData for DebeziumCdcSplit {
+impl<T: CdcSourceTypeTrait> SplitMetaData for DebeziumCdcSplit<T> {
     fn id(&self) -> SplitId {
         assert!(self.mysql_split.is_some() || self.pg_split.is_some());
         if let Some(split) = &self.mysql_split {
@@ -177,11 +151,12 @@ impl SplitMetaData for DebeziumCdcSplit {
     }
 }
 
-impl DebeziumCdcSplit {
+impl<T: CdcSourceTypeTrait> DebeziumCdcSplit<T> {
     pub fn new(mysql_split: Option<MySqlCdcSplit>, pg_split: Option<PostgresCdcSplit>) -> Self {
         Self {
             mysql_split,
             pg_split,
+            _phantom: PhantomData,
         }
     }
 
diff --git a/src/connector/src/source/common.rs b/src/connector/src/source/common.rs
new file mode 100644
index 0000000000000..02f1cbde3de38
--- /dev/null
+++ b/src/connector/src/source/common.rs
@@ -0,0 +1,76 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use futures::{Stream, StreamExt, TryStreamExt};
+use futures_async_stream::try_stream;
+use risingwave_common::error::RwError;
+
+use crate::parser::ParserConfig;
+use crate::source::{SourceContextRef, SourceMessage, SplitReader, StreamChunkWithState};
+
+pub(crate) trait CommonSplitReader: SplitReader + 'static {
+    fn into_data_stream(
+        self,
+    ) -> impl Stream<Item = Result<Vec<SourceMessage>, anyhow::Error>> + Send;
+}
+
+#[try_stream(boxed, ok = StreamChunkWithState, error = RwError)]
+pub(crate) async fn into_chunk_stream(
+    reader: impl CommonSplitReader + Send,
+    parser_config: ParserConfig,
+    source_ctx: SourceContextRef,
+) {
+    let actor_id = source_ctx.source_info.actor_id.to_string();
+    let source_id = source_ctx.source_info.source_id.to_string();
+    let metrics = source_ctx.metrics.clone();
+
+    let data_stream = reader.into_data_stream();
+
+    let data_stream = data_stream
+        .inspect_ok(move |data_batch| {
+            let mut by_split_id = std::collections::HashMap::new();
+
+            for msg in data_batch {
+                by_split_id
+                    .entry(msg.split_id.as_ref())
+                    .or_insert_with(Vec::new)
+                    .push(msg);
+            }
+
+            for (split_id, msgs) in by_split_id {
+                metrics
+                    .partition_input_count
+                    .with_label_values(&[&actor_id, &source_id, split_id])
+                    .inc_by(msgs.len() as u64);
+
+                let sum_bytes = msgs
+                    .iter()
+                    .flat_map(|msg| msg.payload.as_ref().map(|p| p.len() as u64))
+                    .sum();
+
+                metrics
+                    .partition_input_bytes
+                    .with_label_values(&[&actor_id, &source_id, split_id])
+                    .inc_by(sum_bytes);
+            }
+        })
+        .boxed();
+
+    let parser =
+        crate::parser::ByteStreamSourceParserImpl::create(parser_config, source_ctx).await?;
+    #[for_await]
+    for msg_batch in parser.into_stream(data_stream) {
+        yield msg_batch?;
+    }
+}
diff --git a/src/connector/src/source/data_gen_util.rs b/src/connector/src/source/data_gen_util.rs
index 1780807bdef51..7d990cf0ff8c9 100644
--- a/src/connector/src/source/data_gen_util.rs
+++ b/src/connector/src/source/data_gen_util.rs
@@ -19,8 +19,12 @@ use tokio::runtime::Runtime;
 use tokio::sync::mpsc;
 
 /// Spawn the data generator to a dedicated runtime, returns a channel receiver
-/// for acquiring the generated data. This is used for the [`DatagenSplitReader`] and
-/// [`NexmarkSplitReader`] in case that they are CPU intensive and may block the streaming actors.
+/// for acquiring the generated data. This is used for the [`DatagenSplitReader`]
+/// and [`NexmarkSplitReader`] in case that they are CPU intensive
+/// and may block the streaming actors.
+///
+/// [`DatagenSplitReader`]: super::datagen::DatagenSplitReader
+/// [`NexmarkSplitReader`]: super::nexmark::source::reader::NexmarkSplitReader
 pub fn spawn_data_generation_stream<T: Send + 'static>(
     stream: impl Stream<Item = T> + Send + 'static,
     buffer_size: usize,
diff --git a/src/connector/src/source/datagen/source/reader.rs b/src/connector/src/source/datagen/source/reader.rs
index d040a3cb63f21..30d1bfae10c4e 100644
--- a/src/connector/src/source/datagen/source/reader.rs
+++ b/src/connector/src/source/datagen/source/reader.rs
@@ -16,23 +16,20 @@ use std::collections::HashMap;
 
 use anyhow::{anyhow, Result};
 use async_trait::async_trait;
-use futures::{StreamExt, TryStreamExt};
-use futures_async_stream::try_stream;
+use futures::{Stream, StreamExt, TryStreamExt};
 use risingwave_common::field_generator::{FieldGeneratorImpl, VarcharProperty};
 
 use super::generator::DatagenEventGenerator;
-use crate::impl_common_split_reader_logic;
 use crate::parser::{EncodingProperties, ParserConfig, ProtocolProperties};
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
 use crate::source::data_gen_util::spawn_data_generation_stream;
 use crate::source::datagen::source::SEQUENCE_FIELD_KIND;
 use crate::source::datagen::{DatagenProperties, DatagenSplit, FieldDesc};
 use crate::source::{
-    BoxSourceStream, BoxSourceWithStateStream, Column, DataType, SourceContextRef, SplitId,
+    BoxSourceWithStateStream, Column, DataType, SourceContextRef, SourceMessage, SplitId,
     SplitImpl, SplitMetaData, SplitReader,
 };
 
-impl_common_split_reader_logic!(DatagenSplitReader, DatagenProperties);
-
 pub struct DatagenSplitReader {
     generator: DatagenEventGenerator,
     assigned_split: DatagenSplit,
@@ -170,13 +167,17 @@ impl SplitReader for DatagenSplitReader {
                 )
                 .boxed()
             }
-            _ => self.into_chunk_stream(),
+            _ => {
+                let parser_config = self.parser_config.clone();
+                let source_context = self.source_ctx.clone();
+                into_chunk_stream(self, parser_config, source_context)
+            }
         }
     }
 }
 
-impl DatagenSplitReader {
-    pub(crate) fn into_data_stream(self) -> BoxSourceStream {
+impl CommonSplitReader for DatagenSplitReader {
+    fn into_data_stream(self) -> impl Stream<Item = Result<Vec<SourceMessage>, anyhow::Error>> {
         // Will buffer at most 4 event chunks.
         const BUFFER_SIZE: usize = 4;
         spawn_data_generation_stream(self.generator.into_msg_stream(), BUFFER_SIZE).boxed()
diff --git a/src/connector/src/source/external.rs b/src/connector/src/source/external.rs
new file mode 100644
index 0000000000000..3b66a06132b26
--- /dev/null
+++ b/src/connector/src/source/external.rs
@@ -0,0 +1,631 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::future::Future;
+
+use anyhow::anyhow;
+use futures::stream::BoxStream;
+use futures::{pin_mut, StreamExt};
+use futures_async_stream::try_stream;
+use itertools::Itertools;
+use mysql_async::prelude::*;
+use mysql_common::params::Params;
+use mysql_common::value::Value;
+use risingwave_common::bail;
+use risingwave_common::catalog::{Schema, OFFSET_COLUMN_NAME};
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::DataType;
+use risingwave_common::util::iter_util::ZipEqFast;
+use serde_derive::{Deserialize, Serialize};
+
+use crate::error::ConnectorError;
+use crate::parser::mysql_row_to_datums;
+use crate::source::MockExternalTableReader;
+
+pub type ConnectorResult<T> = std::result::Result<T, ConnectorError>;
+
+#[derive(Debug)]
+pub enum ExternalTableType {
+    Undefined,
+    MySql,
+    Postgres,
+    Citus,
+}
+
+impl ExternalTableType {
+    pub fn from_properties(properties: &HashMap<String, String>) -> Self {
+        let connector = properties
+            .get("connector")
+            .map(|c| c.to_ascii_lowercase())
+            .unwrap_or_default();
+        match connector.as_str() {
+            "mysql-cdc" => Self::MySql,
+            "postgres-cdc" => Self::Postgres,
+            "citus-cdc" => Self::Citus,
+            _ => Self::Undefined,
+        }
+    }
+
+    pub fn can_backfill(&self) -> bool {
+        matches!(self, Self::MySql)
+    }
+
+    pub fn create_table_reader(
+        &self,
+        properties: HashMap<String, String>,
+        schema: Schema,
+    ) -> ConnectorResult<ExternalTableReaderImpl> {
+        match self {
+            Self::MySql => Ok(ExternalTableReaderImpl::MySql(
+                MySqlExternalTableReader::new(properties, schema)?,
+            )),
+            _ => bail!(ConnectorError::Config(anyhow!(
+                "invalid external table type: {:?}",
+                *self
+            ))),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct SchemaTableName {
+    pub schema_name: String,
+    pub table_name: String,
+}
+
+const TABLE_NAME_KEY: &str = "table.name";
+const SCHEMA_NAME_KEY: &str = "schema.name";
+
+impl SchemaTableName {
+    pub fn new(schema_name: String, table_name: String) -> Self {
+        Self {
+            schema_name,
+            table_name,
+        }
+    }
+
+    pub fn from_properties(properties: &HashMap<String, String>) -> Self {
+        let table_name = properties
+            .get(TABLE_NAME_KEY)
+            .map(|c| c.to_ascii_lowercase())
+            .unwrap_or_default();
+
+        let schema_name = properties
+            .get(SCHEMA_NAME_KEY)
+            .map(|c| c.to_ascii_lowercase())
+            .unwrap_or_default();
+
+        Self {
+            schema_name,
+            table_name,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default, PartialEq, PartialOrd)]
+pub struct MySqlOffset {
+    pub filename: String,
+    pub position: u64,
+}
+
+impl MySqlOffset {
+    pub fn new(filename: String, position: u64) -> Self {
+        Self { filename, position }
+    }
+}
+
+#[derive(Debug, Clone, Default, PartialEq, PartialOrd)]
+pub struct PostgresOffset {
+    pub txid: u64,
+    pub lsn: u64,
+    pub tx_usec: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, PartialOrd)]
+pub enum CdcOffset {
+    MySql(MySqlOffset),
+    Postgres(PostgresOffset),
+}
+
+// Example debezium offset for Postgres:
+// {
+//     "sourcePartition":
+//     {
+//         "server": "RW_CDC_1004"
+//     },
+//     "sourceOffset":
+//     {
+//         "last_snapshot_record": false,
+//         "lsn": 29973552,
+//         "txId": 1046,
+//         "ts_usec": 1670826189008456,
+//         "snapshot": true
+//     }
+// }
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DebeziumOffset {
+    #[serde(rename = "sourcePartition")]
+    pub source_partition: HashMap<String, String>,
+    #[serde(rename = "sourceOffset")]
+    pub source_offset: DebeziumSourceOffset,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct DebeziumSourceOffset {
+    // postgres snapshot progress
+    pub last_snapshot_record: Option<bool>,
+    // mysql snapshot progress
+    pub snapshot: Option<bool>,
+
+    // mysql binlog offset
+    pub file: Option<String>,
+    pub pos: Option<u64>,
+
+    // postgres binlog offset
+    pub lsn: Option<u64>,
+    #[serde(rename = "txId")]
+    pub txid: Option<u64>,
+    pub tx_usec: Option<u64>,
+}
+
+impl MySqlOffset {
+    pub fn parse_str(offset: &str) -> ConnectorResult<Self> {
+        let dbz_offset: DebeziumOffset = serde_json::from_str(offset).map_err(|e| {
+            ConnectorError::Internal(anyhow!("invalid upstream offset: {}, error: {}", offset, e))
+        })?;
+
+        Ok(Self {
+            filename: dbz_offset
+                .source_offset
+                .file
+                .ok_or_else(|| anyhow!("binlog file not found in offset"))?,
+            position: dbz_offset
+                .source_offset
+                .pos
+                .ok_or_else(|| anyhow!("binlog position not found in offset"))?,
+        })
+    }
+}
+
+pub trait ExternalTableReader {
+    type CdcOffsetFuture<'a>: Future<Output = ConnectorResult<CdcOffset>> + Send + 'a
+    where
+        Self: 'a;
+
+    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String;
+
+    fn current_cdc_offset(&self) -> Self::CdcOffsetFuture<'_>;
+
+    fn parse_binlog_offset(&self, offset: &str) -> ConnectorResult<CdcOffset>;
+
+    fn snapshot_read(
+        &self,
+        table_name: SchemaTableName,
+        start_pk: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+    ) -> BoxStream<'_, ConnectorResult<OwnedRow>>;
+}
+
+#[derive(Debug)]
+pub enum ExternalTableReaderImpl {
+    MySql(MySqlExternalTableReader),
+    Mock(MockExternalTableReader),
+}
+
+#[derive(Debug)]
+pub struct MySqlExternalTableReader {
+    pool: mysql_async::Pool,
+    config: ExternalTableConfig,
+    rw_schema: Schema,
+    field_names: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct ExternalTableConfig {
+    #[serde(rename = "hostname")]
+    pub host: String,
+    pub port: String,
+    pub username: String,
+    pub password: String,
+    #[serde(rename = "database.name")]
+    pub database: String,
+    #[serde(rename = "schema.name", default = "Default::default")]
+    pub schema: String,
+    #[serde(rename = "table.name")]
+    pub table: String,
+}
+
+impl ExternalTableReader for MySqlExternalTableReader {
+    type CdcOffsetFuture<'a> = impl Future<Output = ConnectorResult<CdcOffset>> + 'a;
+
+    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String {
+        format!("`{}`", table_name.table_name)
+    }
+
+    fn current_cdc_offset(&self) -> Self::CdcOffsetFuture<'_> {
+        async move {
+            let mut conn = self
+                .pool
+                .get_conn()
+                .await
+                .map_err(|e| ConnectorError::Connection(anyhow!(e)))?;
+
+            let sql = "SHOW MASTER STATUS".to_string();
+            let mut rs = conn.query::<mysql_async::Row, _>(sql).await?;
+            let row = rs
+                .iter_mut()
+                .exactly_one()
+                .map_err(|e| ConnectorError::Internal(anyhow!("read binlog error: {}", e)))?;
+
+            Ok(CdcOffset::MySql(MySqlOffset {
+                filename: row.take("File").unwrap(),
+                position: row.take("Position").unwrap(),
+            }))
+        }
+    }
+
+    fn parse_binlog_offset(&self, offset: &str) -> ConnectorResult<CdcOffset> {
+        Ok(CdcOffset::MySql(MySqlOffset::parse_str(offset)?))
+    }
+
+    fn snapshot_read(
+        &self,
+        table_name: SchemaTableName,
+        start_pk: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+    ) -> BoxStream<'_, ConnectorResult<OwnedRow>> {
+        self.snapshot_read_inner(table_name, start_pk, primary_keys)
+    }
+}
+
+impl MySqlExternalTableReader {
+    pub fn new(properties: HashMap<String, String>, rw_schema: Schema) -> ConnectorResult<Self> {
+        if let Some(field) = rw_schema.fields.last() && field.name.as_str() != OFFSET_COLUMN_NAME {
+            return Err(ConnectorError::Config(anyhow!(
+                "last column of schema must be `_rw_offset`"
+            )));
+        }
+
+        let config = serde_json::from_value::<ExternalTableConfig>(
+            serde_json::to_value(properties).unwrap(),
+        )
+        .map_err(|e| {
+            ConnectorError::Config(anyhow!("fail to extract mysql connector properties: {}", e))
+        })?;
+
+        let database_url = format!(
+            "mysql://{}:{}@{}:{}/{}",
+            config.username, config.password, config.host, config.port, config.database
+        );
+        let pool = mysql_async::Pool::from_url(database_url)?;
+
+        let field_names = rw_schema
+            .fields
+            .iter()
+            .filter(|f| f.name != OFFSET_COLUMN_NAME)
+            .map(|f| format!("`{}`", f.name.as_str()))
+            .join(",");
+
+        Ok(Self {
+            pool,
+            config,
+            rw_schema,
+            field_names,
+        })
+    }
+
+    pub async fn disconnect(&self) -> ConnectorResult<()> {
+        self.pool.clone().disconnect().await?;
+        Ok(())
+    }
+
+    #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
+    async fn snapshot_read_inner(
+        &self,
+        table_name: SchemaTableName,
+        start_pk_row: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+    ) {
+        let order_key = primary_keys.iter().join(",");
+        let sql = if start_pk_row.is_none() {
+            format!(
+                "SELECT {} FROM {} ORDER BY {}",
+                self.field_names,
+                self.get_normalized_table_name(&table_name),
+                order_key
+            )
+        } else {
+            let filter_expr = Self::filter_expression(&primary_keys);
+            format!(
+                "SELECT {} FROM {} WHERE {} ORDER BY {}",
+                self.field_names,
+                self.get_normalized_table_name(&table_name),
+                filter_expr,
+                order_key
+            )
+        };
+
+        let mut conn = self
+            .pool
+            .get_conn()
+            .await
+            .map_err(|e| ConnectorError::Connection(anyhow!(e)))?;
+
+        // Set session timezone to UTC
+        conn.exec_drop("SET time_zone = \"+00:00\"", ()).await?;
+
+        if start_pk_row.is_none() {
+            let mut result_set = conn.query_iter(sql).await?;
+            let rs_stream = result_set.stream::<mysql_async::Row>().await?;
+            if let Some(rs_stream) = rs_stream {
+                let row_stream = rs_stream.map(|row| {
+                    // convert mysql row into OwnedRow
+                    let mut row = row?;
+                    let datums = mysql_row_to_datums(&mut row, &self.rw_schema);
+                    Ok::<_, ConnectorError>(OwnedRow::new(datums))
+                });
+
+                pin_mut!(row_stream);
+                #[for_await]
+                for row in row_stream {
+                    let row = row?;
+                    yield row;
+                }
+            }
+        } else {
+            let field_map = self
+                .rw_schema
+                .fields
+                .iter()
+                .map(|f| (f.name.as_str(), f.data_type.clone()))
+                .collect::<HashMap<_, _>>();
+
+            // fill in start primary key params
+            let params: Vec<_> = primary_keys
+                .iter()
+                .zip_eq_fast(start_pk_row.unwrap().into_iter())
+                .map(|(pk, datum)| {
+                    if let Some(value) = datum {
+                        let ty = field_map.get(pk.as_str()).unwrap();
+                        let val = match ty {
+                            DataType::Boolean => Value::from(value.into_bool()),
+                            DataType::Int16 => Value::from(value.into_int16()),
+                            DataType::Int32 => Value::from(value.into_int32()),
+                            DataType::Int64 => Value::from(value.into_int64()),
+                            DataType::Float32 => Value::from(value.into_float32().into_inner()),
+                            DataType::Float64 => Value::from(value.into_float64().into_inner()),
+                            DataType::Varchar => Value::from(String::from(value.into_utf8())),
+                            DataType::Date => Value::from(value.into_date().0),
+                            DataType::Time => Value::from(value.into_time().0),
+                            DataType::Timestamp => Value::from(value.into_timestamp().0),
+                            _ => {
+                                return Err(ConnectorError::Internal(anyhow!(
+                                    "unsupported primary key data type: {}",
+                                    ty
+                                )))
+                            }
+                        };
+                        Ok((pk.clone(), val))
+                    } else {
+                        Err(ConnectorError::Internal(anyhow!(
+                            "primary key {} cannot be null",
+                            pk
+                        )))
+                    }
+                })
+                .try_collect()?;
+
+            let rs_stream = sql
+                .with(Params::from(params))
+                .stream::<mysql_async::Row, _>(&mut conn)
+                .await?;
+
+            let row_stream = rs_stream.map(|row| {
+                // convert mysql row into OwnedRow
+                let mut row = row?;
+                let datums = mysql_row_to_datums(&mut row, &self.rw_schema);
+                Ok::<_, ConnectorError>(OwnedRow::new(datums))
+            });
+
+            pin_mut!(row_stream);
+            #[for_await]
+            for row in row_stream {
+                let row = row?;
+                yield row;
+            }
+        };
+    }
+
+    // mysql cannot leverage the given key to narrow down the range of scan,
+    // we need to rewrite the comparison conditions by our own.
+    // (a, b) > (x, y) => ('a' > x) OR (('a' = x) AND ('b' > y))
+    fn filter_expression(columns: &[String]) -> String {
+        let mut conditions = vec![];
+        // push the first condition
+        conditions.push(format!("({} > :{})", columns[0], columns[0]));
+        for i in 2..=columns.len() {
+            // '=' condition
+            let mut condition = String::new();
+            for (j, item) in columns.iter().enumerate().take(i - 1) {
+                if j == 0 {
+                    condition.push_str(&format!("({} = :{})", item, item));
+                } else {
+                    condition.push_str(&format!(" AND ({} = :{})", item, item));
+                }
+            }
+            // '>' condition
+            condition.push_str(&format!(" AND ({} > :{})", columns[i - 1], columns[i - 1]));
+            conditions.push(format!("({})", condition));
+        }
+        if columns.len() > 1 {
+            conditions.join(" OR ")
+        } else {
+            conditions.join("")
+        }
+    }
+}
+
+impl ExternalTableReader for ExternalTableReaderImpl {
+    type CdcOffsetFuture<'a> = impl Future<Output = ConnectorResult<CdcOffset>> + 'a;
+
+    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String {
+        match self {
+            ExternalTableReaderImpl::MySql(mysql) => mysql.get_normalized_table_name(table_name),
+            ExternalTableReaderImpl::Mock(mock) => mock.get_normalized_table_name(table_name),
+        }
+    }
+
+    fn current_cdc_offset(&self) -> Self::CdcOffsetFuture<'_> {
+        async move {
+            match self {
+                ExternalTableReaderImpl::MySql(mysql) => mysql.current_cdc_offset().await,
+                ExternalTableReaderImpl::Mock(mock) => mock.current_cdc_offset().await,
+            }
+        }
+    }
+
+    fn parse_binlog_offset(&self, offset: &str) -> ConnectorResult<CdcOffset> {
+        match self {
+            ExternalTableReaderImpl::MySql(mysql) => mysql.parse_binlog_offset(offset),
+            ExternalTableReaderImpl::Mock(mock) => mock.parse_binlog_offset(offset),
+        }
+    }
+
+    fn snapshot_read(
+        &self,
+        table_name: SchemaTableName,
+        start_pk: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+    ) -> BoxStream<'_, ConnectorResult<OwnedRow>> {
+        self.snapshot_read_inner(table_name, start_pk, primary_keys)
+    }
+}
+
+impl ExternalTableReaderImpl {
+    #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
+    async fn snapshot_read_inner(
+        &self,
+        table_name: SchemaTableName,
+        start_pk: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+    ) {
+        let stream = match self {
+            ExternalTableReaderImpl::MySql(mysql) => {
+                mysql.snapshot_read(table_name, start_pk, primary_keys)
+            }
+            ExternalTableReaderImpl::Mock(mock) => {
+                mock.snapshot_read(table_name, start_pk, primary_keys)
+            }
+        };
+
+        pin_mut!(stream);
+        #[for_await]
+        for row in stream {
+            let row = row?;
+            yield row;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use futures::pin_mut;
+    use futures_async_stream::for_await;
+    use maplit::{convert_args, hashmap};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId};
+    use risingwave_common::types::DataType;
+
+    use crate::sink::catalog::SinkType;
+    use crate::sink::SinkParam;
+    use crate::source::external::{
+        CdcOffset, ExternalTableReader, MySqlExternalTableReader, MySqlOffset, SchemaTableName,
+    };
+
+    #[test]
+    fn test_mysql_filter_expr() {
+        let cols = vec!["aa".to_string(), "bb".to_string(), "cc".to_string()];
+        let expr = MySqlExternalTableReader::filter_expression(&cols);
+        assert_eq!(
+            expr,
+            "(aa > :aa) OR ((aa = :aa) AND (bb > :bb)) OR ((aa = :aa) AND (bb = :bb) AND (cc > :cc))"
+        );
+    }
+
+    #[test]
+    fn test_mysql_binlog_offset() {
+        let off0_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000001", "pos": 105622, "snapshot": true } }"#;
+        let off1_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000007", "pos": 1062363217, "snapshot": true } }"#;
+        let off2_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000007", "pos": 659687560, "snapshot": true } }"#;
+        let off3_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000008", "pos": 7665875, "snapshot": true } }"#;
+        let off4_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000008", "pos": 7665875, "snapshot": true } }"#;
+
+        let off0 = CdcOffset::MySql(MySqlOffset::parse_str(off0_str).unwrap());
+        let off1 = CdcOffset::MySql(MySqlOffset::parse_str(off1_str).unwrap());
+        let off2 = CdcOffset::MySql(MySqlOffset::parse_str(off2_str).unwrap());
+        let off3 = CdcOffset::MySql(MySqlOffset::parse_str(off3_str).unwrap());
+        let off4 = CdcOffset::MySql(MySqlOffset::parse_str(off4_str).unwrap());
+
+        assert!(off0 <= off1);
+        assert!(off1 > off2);
+        assert!(off2 < off3);
+        assert_eq!(off3, off4);
+    }
+
+    // manual test case
+    #[ignore]
+    #[tokio::test]
+    async fn test_mysql_table_reader() {
+        let param = SinkParam {
+            sink_id: Default::default(),
+            properties: Default::default(),
+            columns: vec![
+                ColumnDesc::unnamed(ColumnId::new(1), DataType::Int32),
+                ColumnDesc::unnamed(ColumnId::new(2), DataType::Decimal),
+                ColumnDesc::unnamed(ColumnId::new(3), DataType::Varchar),
+                ColumnDesc::unnamed(ColumnId::new(4), DataType::Date),
+            ],
+            pk_indices: vec![0],
+            sink_type: SinkType::AppendOnly,
+            db_name: "db".into(),
+            sink_from_name: "table".into(),
+        };
+
+        let rw_schema = param.schema();
+        let props = convert_args!(hashmap!(
+                "hostname" => "localhost",
+                "port" => "8306",
+                "username" => "root",
+                "password" => "123456",
+                "database.name" => "mydb",
+                "table.name" => "t1"));
+
+        let reader = MySqlExternalTableReader::new(props, rw_schema).unwrap();
+        let offset = reader.current_cdc_offset().await.unwrap();
+        println!("BinlogOffset: {:?}", offset);
+
+        let table_name = SchemaTableName {
+            schema_name: "mydb".to_string(),
+            table_name: "t1".to_string(),
+        };
+
+        let stream = reader.snapshot_read(table_name, None, vec!["v1".to_string()]);
+        pin_mut!(stream);
+        #[for_await]
+        for row in stream {
+            println!("OwnedRow: {:?}", row);
+        }
+    }
+}
diff --git a/src/connector/src/source/google_pubsub/mod.rs b/src/connector/src/source/google_pubsub/mod.rs
index 0619edde5a8bd..0c93f672ccdb5 100644
--- a/src/connector/src/source/google_pubsub/mod.rs
+++ b/src/connector/src/source/google_pubsub/mod.rs
@@ -39,7 +39,7 @@ pub struct PubsubProperties {
     pub subscription: String,
 
     /// use the connector with a pubsub emulator
-    /// https://cloud.google.com/pubsub/docs/emulator
+    /// <https://cloud.google.com/pubsub/docs/emulator>
     #[serde(rename = "pubsub.emulator_host")]
     pub emulator_host: Option<String>,
 
diff --git a/src/connector/src/source/google_pubsub/source/reader.rs b/src/connector/src/source/google_pubsub/source/reader.rs
index f5d4955ec9b20..d4fa8a9ab5c98 100644
--- a/src/connector/src/source/google_pubsub/source/reader.rs
+++ b/src/connector/src/source/google_pubsub/source/reader.rs
@@ -15,7 +15,6 @@
 use anyhow::{anyhow, ensure, Context, Result};
 use async_trait::async_trait;
 use chrono::{NaiveDateTime, TimeZone, Utc};
-use futures::{StreamExt, TryStreamExt};
 use futures_async_stream::try_stream;
 use google_cloud_pubsub::client::Client;
 use google_cloud_pubsub::subscription::{SeekTo, Subscription};
@@ -23,8 +22,8 @@ use risingwave_common::bail;
 use tonic::Code;
 
 use super::TaggedReceivedMessage;
-use crate::impl_common_split_reader_logic;
 use crate::parser::ParserConfig;
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
 use crate::source::google_pubsub::PubsubProperties;
 use crate::source::{
     BoxSourceWithStateStream, Column, SourceContextRef, SourceMessage, SplitId, SplitImpl,
@@ -33,8 +32,6 @@ use crate::source::{
 
 const PUBSUB_MAX_FETCH_MESSAGES: usize = 1024;
 
-impl_common_split_reader_logic!(PubsubSplitReader, PubsubProperties);
-
 pub struct PubsubSplitReader {
     subscription: Subscription,
     stop_offset: Option<NaiveDateTime>,
@@ -44,8 +41,8 @@ pub struct PubsubSplitReader {
     source_ctx: SourceContextRef,
 }
 
-impl PubsubSplitReader {
-    #[try_stream(boxed, ok = Vec<SourceMessage>, error = anyhow::Error)]
+impl CommonSplitReader for PubsubSplitReader {
+    #[try_stream(ok = Vec<SourceMessage>, error = anyhow::Error)]
     async fn into_data_stream(self) {
         loop {
             let pull_result = self
@@ -172,6 +169,8 @@ impl SplitReader for PubsubSplitReader {
     }
 
     fn into_stream(self) -> BoxSourceWithStateStream {
-        self.into_chunk_stream()
+        let parser_config = self.parser_config.clone();
+        let source_context = self.source_ctx.clone();
+        into_chunk_stream(self, parser_config, source_context)
     }
 }
diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs
index d443429857c21..103255a882ed7 100644
--- a/src/connector/src/source/kafka/source/reader.rs
+++ b/src/connector/src/source/kafka/source/reader.rs
@@ -19,16 +19,16 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{anyhow, Result};
 use async_trait::async_trait;
-use futures::{StreamExt, TryStreamExt};
+use futures::StreamExt;
 use futures_async_stream::try_stream;
 use rdkafka::config::RDKafkaLogLevel;
 use rdkafka::consumer::{Consumer, StreamConsumer};
 use rdkafka::error::KafkaError;
 use rdkafka::{ClientConfig, Message, Offset, TopicPartitionList};
 
-use crate::impl_common_split_reader_logic;
 use crate::parser::ParserConfig;
 use crate::source::base::SourceMessage;
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
 use crate::source::kafka::{
     KafkaProperties, KafkaSplit, PrivateLinkConsumerContext, KAFKA_ISOLATION_LEVEL,
 };
@@ -37,8 +37,6 @@ use crate::source::{
     SplitReader,
 };
 
-impl_common_split_reader_logic!(KafkaSplitReader, KafkaProperties);
-
 pub struct KafkaSplitReader {
     consumer: StreamConsumer<PrivateLinkConsumerContext>,
     offsets: HashMap<SplitId, (Option<i64>, Option<i64>)>,
@@ -159,7 +157,9 @@ impl SplitReader for KafkaSplitReader {
     }
 
     fn into_stream(self) -> BoxSourceWithStateStream {
-        self.into_chunk_stream()
+        let parser_config = self.parser_config.clone();
+        let source_context = self.source_ctx.clone();
+        into_chunk_stream(self, parser_config, source_context)
     }
 }
 
@@ -176,9 +176,11 @@ impl KafkaSplitReader {
             ])
             .set(offset);
     }
+}
 
-    #[try_stream(boxed, ok = Vec<SourceMessage>, error = anyhow::Error)]
-    pub async fn into_data_stream(self) {
+impl CommonSplitReader for KafkaSplitReader {
+    #[try_stream(ok = Vec<SourceMessage>, error = anyhow::Error)]
+    async fn into_data_stream(self) {
         if self.offsets.values().all(|(start_offset, stop_offset)| {
             match (start_offset, stop_offset) {
                 (Some(start), Some(stop)) if (*start + 1) >= *stop => true,
diff --git a/src/connector/src/source/kinesis/enumerator/client.rs b/src/connector/src/source/kinesis/enumerator/client.rs
index 9f1c49e62c2da..59d1aca06e4f1 100644
--- a/src/connector/src/source/kinesis/enumerator/client.rs
+++ b/src/connector/src/source/kinesis/enumerator/client.rs
@@ -76,6 +76,7 @@ impl SplitEnumerator for KinesisSplitEnumerator {
             .into_iter()
             .map(|x| KinesisSplit {
                 shard_id: x.shard_id().unwrap_or_default().to_string().into(),
+                // handle start with position in reader part
                 start_position: KinesisOffset::None,
                 end_position: KinesisOffset::None,
             })
diff --git a/src/connector/src/source/kinesis/source/reader.rs b/src/connector/src/source/kinesis/source/reader.rs
index 8d714f3b79334..f4aad6ad80587 100644
--- a/src/connector/src/source/kinesis/source/reader.rs
+++ b/src/connector/src/source/kinesis/source/reader.rs
@@ -20,12 +20,11 @@ use aws_sdk_kinesis::error::{DisplayErrorContext, SdkError};
 use aws_sdk_kinesis::operation::get_records::{GetRecordsError, GetRecordsOutput};
 use aws_sdk_kinesis::types::ShardIteratorType;
 use aws_sdk_kinesis::Client as KinesisClient;
-use futures::{StreamExt, TryStreamExt};
 use futures_async_stream::try_stream;
 use tokio_retry;
 
-use crate::impl_common_split_reader_logic;
 use crate::parser::ParserConfig;
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
 use crate::source::kinesis::source::message::KinesisMessage;
 use crate::source::kinesis::split::KinesisOffset;
 use crate::source::kinesis::KinesisProperties;
@@ -34,8 +33,6 @@ use crate::source::{
     SplitMetaData, SplitReader,
 };
 
-impl_common_split_reader_logic!(KinesisSplitReader, KinesisProperties);
-
 #[derive(Debug, Clone)]
 pub struct KinesisSplitReader {
     client: KinesisClient,
@@ -89,6 +86,14 @@ impl SplitReader for KinesisSplitReader {
             start_position => start_position.to_owned(),
         };
 
+        if !matches!(start_position, KinesisOffset::SequenceNumber(_))
+            && properties.seq_offset.is_some()
+        {
+            return Err(
+                anyhow!("scan.startup.mode need to be set to 'sequence_number' if you want to start with a specific sequence number")
+            );
+        }
+
         let stream_name = properties.common.stream_name.clone();
         let client = properties.common.build_client().await?;
 
@@ -108,13 +113,15 @@ impl SplitReader for KinesisSplitReader {
     }
 
     fn into_stream(self) -> BoxSourceWithStateStream {
-        self.into_chunk_stream()
+        let parser_config = self.parser_config.clone();
+        let source_context = self.source_ctx.clone();
+        into_chunk_stream(self, parser_config, source_context)
     }
 }
 
-impl KinesisSplitReader {
-    #[try_stream(boxed, ok = Vec<SourceMessage>, error = anyhow::Error)]
-    pub(crate) async fn into_data_stream(mut self) {
+impl CommonSplitReader for KinesisSplitReader {
+    #[try_stream(ok = Vec < SourceMessage >, error = anyhow::Error)]
+    async fn into_data_stream(mut self) {
         self.new_shard_iter().await?;
         loop {
             if self.shard_iter.is_none() {
@@ -185,11 +192,21 @@ impl KinesisSplitReader {
                     self.new_shard_iter().await?;
                     continue;
                 }
-                Err(e) => return Err(anyhow!(DisplayErrorContext(e))),
+                Err(e) => {
+                    let error_msg = format!(
+                        "Kinesis got a unhandled error: {:?}, stream {:?}, shard {:?}",
+                        DisplayErrorContext(e),
+                        self.stream_name,
+                        self.shard_id,
+                    );
+                    tracing::error!("{}", error_msg);
+                    return Err(anyhow!("{}", error_msg));
+                }
             }
         }
     }
-
+}
+impl KinesisSplitReader {
     async fn new_shard_iter(&mut self) -> Result<()> {
         let (starting_seq_num, iter_type) = if self.latest_offset.is_some() {
             (
@@ -269,12 +286,47 @@ impl KinesisSplitReader {
 
 #[cfg(test)]
 mod tests {
-    use futures::StreamExt;
+    use futures::{pin_mut, StreamExt};
 
     use super::*;
     use crate::common::KinesisCommon;
     use crate::source::kinesis::split::KinesisSplit;
 
+    #[tokio::test]
+    async fn test_reject_redundant_seq_props() {
+        let properties = KinesisProperties {
+            common: KinesisCommon {
+                assume_role_arn: None,
+                credentials_access_key: None,
+                credentials_secret_access_key: None,
+                stream_name: "kinesis_debug".to_string(),
+                stream_region: "cn-northwest-1".to_string(),
+                endpoint: None,
+                session_token: None,
+                assume_role_external_id: None,
+            },
+
+            scan_startup_mode: None,
+            seq_offset: Some(
+                // redundant seq number
+                "49629139817504901062972448413535783695568426186596941842".to_string(),
+            ),
+        };
+        let client = KinesisSplitReader::new(
+            properties,
+            vec![SplitImpl::Kinesis(KinesisSplit {
+                shard_id: "shardId-000000000001".to_string().into(),
+                start_position: KinesisOffset::Earliest,
+                end_position: KinesisOffset::None,
+            })],
+            Default::default(),
+            Default::default(),
+            None,
+        )
+        .await;
+        assert!(client.is_err());
+    }
+
     #[tokio::test]
     #[ignore]
     async fn test_single_thread_kinesis_reader() -> Result<()> {
@@ -294,7 +346,7 @@ mod tests {
             seq_offset: None,
         };
 
-        let mut trim_horizen_reader = KinesisSplitReader::new(
+        let trim_horizen_reader = KinesisSplitReader::new(
             properties.clone(),
             vec![SplitImpl::Kinesis(KinesisSplit {
                 shard_id: "shardId-000000000001".to_string().into(),
@@ -307,9 +359,10 @@ mod tests {
         )
         .await?
         .into_data_stream();
+        pin_mut!(trim_horizen_reader);
         println!("{:?}", trim_horizen_reader.next().await.unwrap()?);
 
-        let mut offset_reader = KinesisSplitReader::new(
+        let offset_reader = KinesisSplitReader::new(
             properties.clone(),
             vec![SplitImpl::Kinesis(KinesisSplit {
                 shard_id: "shardId-000000000001".to_string().into(),
@@ -324,6 +377,7 @@ mod tests {
         )
         .await?
         .into_data_stream();
+        pin_mut!(offset_reader);
         println!("{:?}", offset_reader.next().await.unwrap()?);
 
         Ok(())
diff --git a/src/connector/src/source/manager.rs b/src/connector/src/source/manager.rs
index 997cf40c4b8c0..bdd72a090f9e8 100644
--- a/src/connector/src/source/manager.rs
+++ b/src/connector/src/source/manager.rs
@@ -14,7 +14,9 @@
 
 use std::fmt::Debug;
 
-use risingwave_common::catalog::{ColumnDesc, ColumnId, KAFKA_TIMESTAMP_COLUMN_NAME};
+use risingwave_common::catalog::{
+    ColumnDesc, ColumnId, KAFKA_TIMESTAMP_COLUMN_NAME, OFFSET_COLUMN_NAME, ROWID_PREFIX,
+};
 use risingwave_common::types::DataType;
 
 /// `SourceColumnDesc` is used to describe a column in the Source and is used as the column
@@ -25,14 +27,42 @@ pub struct SourceColumnDesc {
     pub data_type: DataType,
     pub column_id: ColumnId,
     pub fields: Vec<ColumnDesc>,
-    /// Now `skip_parse` is used to indicate whether the column is a row id column.
-    pub is_row_id: bool,
+    pub column_type: SourceColumnType,
 
-    pub is_meta: bool,
     // `is_pk` is used to indicate whether the column is part of the primary key columns.
     pub is_pk: bool,
 }
 
+/// `SourceColumnType` is used to indicate the type of a column emitted by the Source.
+/// There are 4 types of columns:
+/// - `Normal`: a visible column
+/// - `RowId`: internal column to uniquely identify a row
+/// - `Meta`: internal column to store source related metadata
+/// - `Offset`: internal column to store upstream offset for a row, used in CDC source
+#[derive(Clone, Debug, PartialEq)]
+pub enum SourceColumnType {
+    Normal,
+
+    // internal columns
+    RowId,
+    Meta,
+    Offset,
+}
+
+impl SourceColumnType {
+    pub fn from_name(name: &str) -> Self {
+        if name.starts_with(KAFKA_TIMESTAMP_COLUMN_NAME) {
+            Self::Meta
+        } else if name == (ROWID_PREFIX) {
+            Self::RowId
+        } else if name == OFFSET_COLUMN_NAME {
+            Self::Offset
+        } else {
+            Self::Normal
+        }
+    }
+}
+
 impl SourceColumnDesc {
     /// Create a [`SourceColumnDesc`] without composite types.
     #[track_caller]
@@ -47,28 +77,38 @@ impl SourceColumnDesc {
             data_type,
             column_id,
             fields: vec![],
-            is_row_id: false,
-            is_meta: false,
+            column_type: SourceColumnType::Normal,
             is_pk: false,
         }
     }
 
+    pub fn is_row_id(&self) -> bool {
+        self.column_type == SourceColumnType::RowId
+    }
+
+    pub fn is_meta(&self) -> bool {
+        self.column_type == SourceColumnType::Meta
+    }
+
+    pub fn is_offset(&self) -> bool {
+        self.column_type == SourceColumnType::Offset
+    }
+
     #[inline]
     pub fn is_visible(&self) -> bool {
-        !self.is_row_id && !self.is_meta
+        self.column_type == SourceColumnType::Normal
     }
 }
 
 impl From<&ColumnDesc> for SourceColumnDesc {
     fn from(c: &ColumnDesc) -> Self {
-        let is_meta = c.name.starts_with(KAFKA_TIMESTAMP_COLUMN_NAME);
+        let column_type = SourceColumnType::from_name(c.name.as_str());
         Self {
             name: c.name.clone(),
             data_type: c.data_type.clone(),
             column_id: c.column_id,
             fields: c.field_descs.clone(),
-            is_row_id: c.name.as_str() == "_row_id",
-            is_meta,
+            column_type,
             is_pk: false,
         }
     }
@@ -95,10 +135,9 @@ mod tests {
     fn test_is_visible() {
         let mut c = SourceColumnDesc::simple("a", DataType::Int32, ColumnId::new(0));
         assert!(c.is_visible());
-        c.is_row_id = true;
+        c.column_type = SourceColumnType::RowId;
         assert!(!c.is_visible());
-        c.is_row_id = false;
-        c.is_meta = true;
+        c.column_type = SourceColumnType::Meta;
         assert!(!c.is_visible());
     }
 }
diff --git a/src/connector/src/source/mock_external_table.rs b/src/connector/src/source/mock_external_table.rs
new file mode 100644
index 0000000000000..c4e16aae6ae85
--- /dev/null
+++ b/src/connector/src/source/mock_external_table.rs
@@ -0,0 +1,128 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+use std::sync::atomic::AtomicUsize;
+
+use futures::stream::BoxStream;
+use futures_async_stream::try_stream;
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::ScalarImpl;
+
+use crate::error::ConnectorError;
+use crate::source::external::{
+    CdcOffset, ConnectorResult, ExternalTableReader, MySqlOffset, SchemaTableName,
+};
+
+#[derive(Debug)]
+pub struct MockExternalTableReader {
+    binlog_watermarks: Vec<MySqlOffset>,
+    snapshot_cnt: AtomicUsize,
+}
+
+impl MockExternalTableReader {
+    pub fn new(binlog_watermarks: Vec<MySqlOffset>) -> Self {
+        Self {
+            binlog_watermarks,
+            snapshot_cnt: AtomicUsize::new(0),
+        }
+    }
+
+    #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
+    async fn snapshot_read_inner(&self) {
+        let snap_idx = self
+            .snapshot_cnt
+            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        println!("snapshot read: idx {}", snap_idx);
+
+        let snap0 = vec![OwnedRow::new(vec![
+            Some(ScalarImpl::Int64(1)),
+            Some(ScalarImpl::Float64(1.0001.into())),
+            None,
+        ])];
+        let snap1 = vec![
+            OwnedRow::new(vec![
+                Some(ScalarImpl::Int64(1)),
+                Some(ScalarImpl::Float64(10.01.into())),
+                None,
+            ]),
+            OwnedRow::new(vec![
+                Some(ScalarImpl::Int64(2)),
+                Some(ScalarImpl::Float64(2.02.into())),
+                None,
+            ]),
+            OwnedRow::new(vec![
+                Some(ScalarImpl::Int64(5)),
+                Some(ScalarImpl::Float64(1.0005.into())),
+                None,
+            ]),
+            OwnedRow::new(vec![
+                Some(ScalarImpl::Int64(6)),
+                Some(ScalarImpl::Float64(1.0006.into())),
+                None,
+            ]),
+            OwnedRow::new(vec![
+                Some(ScalarImpl::Int64(8)),
+                Some(ScalarImpl::Float64(1.0008.into())),
+                None,
+            ]),
+        ];
+
+        let snapshots = vec![snap0, snap1];
+        if snap_idx >= snapshots.len() {
+            return Ok(());
+        }
+
+        for row in &snapshots[snap_idx] {
+            yield row.clone();
+        }
+    }
+}
+
+impl ExternalTableReader for MockExternalTableReader {
+    type CdcOffsetFuture<'a> = impl Future<Output = ConnectorResult<CdcOffset>> + 'a;
+
+    fn get_normalized_table_name(&self, _table_name: &SchemaTableName) -> String {
+        "`mock_table`".to_string()
+    }
+
+    fn current_cdc_offset(&self) -> Self::CdcOffsetFuture<'_> {
+        static IDX: AtomicUsize = AtomicUsize::new(0);
+        async move {
+            let idx = IDX.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            if idx < self.binlog_watermarks.len() {
+                Ok(CdcOffset::MySql(self.binlog_watermarks[idx].clone()))
+            } else {
+                Ok(CdcOffset::MySql(MySqlOffset {
+                    filename: "1.binlog".to_string(),
+                    position: u64::MAX,
+                }))
+            }
+        }
+    }
+
+    fn parse_binlog_offset(&self, offset: &str) -> ConnectorResult<CdcOffset> {
+        // same as mysql offset
+        Ok(CdcOffset::MySql(MySqlOffset::parse_str(offset)?))
+    }
+
+    fn snapshot_read(
+        &self,
+        _table_name: SchemaTableName,
+        _start_pk: Option<OwnedRow>,
+        _primary_keys: Vec<String>,
+    ) -> BoxStream<'_, ConnectorResult<OwnedRow>> {
+        self.snapshot_read_inner()
+    }
+}
diff --git a/src/connector/src/source/mod.rs b/src/connector/src/source/mod.rs
index ac10ded25c688..ed3979ba67ca2 100644
--- a/src/connector/src/source/mod.rs
+++ b/src/connector/src/source/mod.rs
@@ -22,14 +22,20 @@ pub mod google_pubsub;
 pub mod kafka;
 pub mod kinesis;
 pub mod monitor;
+pub mod nats;
 pub mod nexmark;
 pub mod pulsar;
 pub use base::*;
 pub use google_pubsub::GOOGLE_PUBSUB_CONNECTOR;
 pub use kafka::KAFKA_CONNECTOR;
 pub use kinesis::KINESIS_CONNECTOR;
+pub use nats::NATS_CONNECTOR;
+mod common;
+pub mod external;
 mod manager;
-pub use manager::SourceColumnDesc;
+mod mock_external_table;
+pub use manager::{SourceColumnDesc, SourceColumnType};
+pub use mock_external_table::MockExternalTableReader;
 
 pub use crate::source::nexmark::NEXMARK_CONNECTOR;
 pub use crate::source::pulsar::PULSAR_CONNECTOR;
diff --git a/src/connector/src/source/nats/enumerator/mod.rs b/src/connector/src/source/nats/enumerator/mod.rs
new file mode 100644
index 0000000000000..88384bfb685e6
--- /dev/null
+++ b/src/connector/src/source/nats/enumerator/mod.rs
@@ -0,0 +1,53 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow;
+use async_trait::async_trait;
+
+use super::source::NatsSplit;
+use super::NatsProperties;
+use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct NatsSplitEnumerator {
+    subject: String,
+    split_num: i32,
+}
+
+#[async_trait]
+impl SplitEnumerator for NatsSplitEnumerator {
+    type Properties = NatsProperties;
+    type Split = NatsSplit;
+
+    async fn new(
+        properties: Self::Properties,
+        _context: SourceEnumeratorContextRef,
+    ) -> anyhow::Result<NatsSplitEnumerator> {
+        Ok(Self {
+            subject: properties.common.subject,
+            split_num: 0,
+        })
+    }
+
+    async fn list_splits(&mut self) -> anyhow::Result<Vec<NatsSplit>> {
+        // TODO: to simplify the logic, return 1 split for first version
+        let nats_split = NatsSplit {
+            subject: self.subject.clone(),
+            split_num: 0, // be the same as `from_nats_jetstream_message`
+            start_sequence: None,
+        };
+
+        Ok(vec![nats_split])
+    }
+}
diff --git a/src/connector/src/source/nats/mod.rs b/src/connector/src/source/nats/mod.rs
new file mode 100644
index 0000000000000..2aa9dc2de55f2
--- /dev/null
+++ b/src/connector/src/source/nats/mod.rs
@@ -0,0 +1,30 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod enumerator;
+pub mod source;
+pub mod split;
+
+use serde::Deserialize;
+
+use crate::common::NatsCommon;
+pub const NATS_CONNECTOR: &str = "nats";
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct NatsProperties {
+    #[serde(flatten)]
+    pub common: NatsCommon,
+}
+
+impl NatsProperties {}
diff --git a/src/connector/src/source/nats/source/message.rs b/src/connector/src/source/nats/source/message.rs
new file mode 100644
index 0000000000000..afb3029d3b917
--- /dev/null
+++ b/src/connector/src/source/nats/source/message.rs
@@ -0,0 +1,31 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_nats;
+
+use crate::source::base::SourceMessage;
+use crate::source::SourceMeta;
+
+impl SourceMessage {
+    pub fn from_nats_jetstream_message(message: async_nats::jetstream::message::Message) -> Self {
+        SourceMessage {
+            key: None,
+            payload: Some(message.message.payload.to_vec()),
+            // For nats jetstream, use sequence id as offset
+            offset: message.info().unwrap().stream_sequence.to_string(),
+            split_id: "0".into(),
+            meta: SourceMeta::Empty,
+        }
+    }
+}
diff --git a/src/connector/src/source/nats/source/mod.rs b/src/connector/src/source/nats/source/mod.rs
new file mode 100644
index 0000000000000..791003c39ccc5
--- /dev/null
+++ b/src/connector/src/source/nats/source/mod.rs
@@ -0,0 +1,20 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod message;
+mod reader;
+
+pub use reader::*;
+
+pub use crate::source::nats::split::*;
diff --git a/src/connector/src/source/nats/source/reader.rs b/src/connector/src/source/nats/source/reader.rs
new file mode 100644
index 0000000000000..c0070a16c1392
--- /dev/null
+++ b/src/connector/src/source/nats/source/reader.rs
@@ -0,0 +1,86 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::Result;
+use async_nats::jetstream::consumer;
+use async_trait::async_trait;
+use futures::StreamExt;
+use futures_async_stream::try_stream;
+
+use crate::parser::ParserConfig;
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
+use crate::source::nats::split::NatsSplit;
+use crate::source::nats::NatsProperties;
+use crate::source::{
+    BoxSourceWithStateStream, Column, SourceContextRef, SourceMessage, SplitImpl, SplitReader,
+};
+
+pub struct NatsSplitReader {
+    consumer: consumer::Consumer<consumer::pull::Config>,
+    properties: NatsProperties,
+    parser_config: ParserConfig,
+    source_ctx: SourceContextRef,
+}
+
+#[async_trait]
+impl SplitReader for NatsSplitReader {
+    type Properties = NatsProperties;
+
+    async fn new(
+        properties: NatsProperties,
+        splits: Vec<SplitImpl>,
+        parser_config: ParserConfig,
+        source_ctx: SourceContextRef,
+        _columns: Option<Vec<Column>>,
+    ) -> Result<Self> {
+        // TODO: to simplify the logic, return 1 split for first version
+        assert!(splits.len() == 1);
+        let splits = splits
+            .into_iter()
+            .map(|split| split.into_nats().unwrap())
+            .collect::<Vec<NatsSplit>>();
+        let consumer = properties
+            .common
+            .build_consumer(0, splits[0].start_sequence)
+            .await?;
+        Ok(Self {
+            consumer,
+            properties,
+            parser_config,
+            source_ctx,
+        })
+    }
+
+    fn into_stream(self) -> BoxSourceWithStateStream {
+        let parser_config = self.parser_config.clone();
+        let source_context = self.source_ctx.clone();
+        into_chunk_stream(self, parser_config, source_context)
+    }
+}
+
+impl CommonSplitReader for NatsSplitReader {
+    #[try_stream(ok = Vec<SourceMessage>, error = anyhow::Error)]
+    async fn into_data_stream(self) {
+        let capacity = self.source_ctx.source_ctrl_opts.chunk_size;
+        let messages = self.consumer.messages().await?;
+        #[for_await]
+        for msgs in messages.ready_chunks(capacity) {
+            let mut msg_vec = Vec::with_capacity(capacity);
+            for msg in msgs {
+                msg_vec.push(SourceMessage::from_nats_jetstream_message(msg?));
+            }
+            yield msg_vec;
+        }
+    }
+}
diff --git a/src/connector/src/source/nats/split.rs b/src/connector/src/source/nats/split.rs
new file mode 100644
index 0000000000000..f0fcfaff35481
--- /dev/null
+++ b/src/connector/src/source/nats/split.rs
@@ -0,0 +1,59 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::{anyhow, Ok};
+use risingwave_common::types::JsonbVal;
+use serde::{Deserialize, Serialize};
+
+use crate::source::{SplitId, SplitMetaData};
+
+/// The states of a NATS split, which will be persisted to checkpoint.
+#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
+pub struct NatsSplit {
+    pub(crate) subject: String,
+    // TODO: to simplify the logic, return 1 split for first version. May use parallelism in
+    // future.
+    pub(crate) split_num: i32,
+    pub(crate) start_sequence: Option<u64>,
+}
+
+impl SplitMetaData for NatsSplit {
+    fn id(&self) -> SplitId {
+        // TODO: should avoid constructing a string every time
+        format!("{}", self.split_num).into()
+    }
+
+    fn restore_from_json(value: JsonbVal) -> anyhow::Result<Self> {
+        serde_json::from_value(value.take()).map_err(|e| anyhow!(e))
+    }
+
+    fn encode_to_json(&self) -> JsonbVal {
+        serde_json::to_value(self.clone()).unwrap().into()
+    }
+}
+
+impl NatsSplit {
+    pub fn new(subject: String, split_num: i32, start_sequence: Option<u64>) -> Self {
+        Self {
+            subject,
+            split_num,
+            start_sequence,
+        }
+    }
+
+    pub fn update_with_offset(&mut self, start_sequence: String) -> anyhow::Result<()> {
+        self.start_sequence = Some(start_sequence.as_str().parse::<u64>().unwrap());
+        Ok(())
+    }
+}
diff --git a/src/connector/src/source/nexmark/source/reader.rs b/src/connector/src/source/nexmark/source/reader.rs
index e7a9c03b5f9b6..190a3f1cbb63d 100644
--- a/src/connector/src/source/nexmark/source/reader.rs
+++ b/src/connector/src/source/nexmark/source/reader.rs
@@ -88,7 +88,7 @@ impl SplitReader for NexmarkSplitReader {
             .common
             .rw_columns
             .into_iter()
-            .position(|column| column.is_row_id);
+            .position(|column| column.is_row_id());
 
         Ok(NexmarkSplitReader {
             generator,
diff --git a/src/connector/src/source/pulsar/mod.rs b/src/connector/src/source/pulsar/mod.rs
index 4c45e0110fa3d..5d2bbfa332307 100644
--- a/src/connector/src/source/pulsar/mod.rs
+++ b/src/connector/src/source/pulsar/mod.rs
@@ -51,7 +51,7 @@ pub struct PulsarOauth {
     pub scope: Option<String>,
 
     #[serde(flatten)]
-    /// required keys refer to [`AWS_DEFAULT_CONFIG`]
+    /// required keys refer to [`crate::aws_utils::AWS_DEFAULT_CONFIG`]
     pub s3_credentials: HashMap<String, String>,
 }
 
diff --git a/src/connector/src/source/pulsar/source/reader.rs b/src/connector/src/source/pulsar/source/reader.rs
index a8e801541f273..9c749c27d616a 100644
--- a/src/connector/src/source/pulsar/source/reader.rs
+++ b/src/connector/src/source/pulsar/source/reader.rs
@@ -16,7 +16,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 
 use anyhow::{anyhow, ensure, Result};
 use async_trait::async_trait;
-use futures::{StreamExt, TryStreamExt};
+use futures::StreamExt;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use pulsar::consumer::InitialPosition;
@@ -24,8 +24,8 @@ use pulsar::message::proto::MessageIdData;
 use pulsar::{Consumer, ConsumerBuilder, ConsumerOptions, Pulsar, SubType, TokioExecutor};
 use risingwave_common::try_match_expand;
 
-use crate::impl_common_split_reader_logic;
 use crate::parser::ParserConfig;
+use crate::source::common::{into_chunk_stream, CommonSplitReader};
 use crate::source::pulsar::split::PulsarSplit;
 use crate::source::pulsar::{PulsarEnumeratorOffset, PulsarProperties};
 use crate::source::{
@@ -33,8 +33,6 @@ use crate::source::{
     SplitMetaData, SplitReader,
 };
 
-impl_common_split_reader_logic!(PulsarSplitReader, PulsarProperties);
-
 pub struct PulsarSplitReader {
     pulsar: Pulsar<TokioExecutor>,
     consumer: Consumer<Vec<u8>, TokioExecutor>,
@@ -170,13 +168,15 @@ impl SplitReader for PulsarSplitReader {
     }
 
     fn into_stream(self) -> BoxSourceWithStateStream {
-        self.into_chunk_stream()
+        let parser_config = self.parser_config.clone();
+        let source_context = self.source_ctx.clone();
+        into_chunk_stream(self, parser_config, source_context)
     }
 }
 
-impl PulsarSplitReader {
-    #[try_stream(boxed, ok = Vec<SourceMessage>, error = anyhow::Error)]
-    pub(crate) async fn into_data_stream(self) {
+impl CommonSplitReader for PulsarSplitReader {
+    #[try_stream(ok = Vec<SourceMessage>, error = anyhow::Error)]
+    async fn into_data_stream(self) {
         let max_chunk_size = self.source_ctx.source_ctrl_opts.chunk_size;
         #[for_await]
         for msgs in self.consumer.ready_chunks(max_chunk_size) {
diff --git a/src/connector/src/source/pulsar/topic.rs b/src/connector/src/source/pulsar/topic.rs
index 9ce22f6ce6206..ab4b1f84750f3 100644
--- a/src/connector/src/source/pulsar/topic.rs
+++ b/src/connector/src/source/pulsar/topic.rs
@@ -93,10 +93,10 @@ pub fn get_partition_index(topic: &str) -> Result<Option<i32>> {
 
 /// `parse_topic` parses a topic name into its components.
 /// The short topic name can be:
-/// - <topic>
-/// - <tenant>/<namespace>/<topic>
+/// - `<topic>`
+/// - `<tenant>/<namespace>/<topic>`
 /// The fully qualified topic name can be:
-/// <domain>://<tenant>/<namespace>/<topic>
+/// `<domain>://<tenant>/<namespace>/<topic>`
 pub fn parse_topic(topic: &str) -> Result<Topic> {
     let mut complete_topic = topic.to_string();
 
diff --git a/src/connector/src/test_data/complex-schema.json b/src/connector/src/test_data/complex-schema.json
new file mode 100644
index 0000000000000..6617021410779
--- /dev/null
+++ b/src/connector/src/test_data/complex-schema.json
@@ -0,0 +1,47 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://example.com/product.schema.json",
+    "title": "Product",
+    "description": "A product from Acme's catalog",
+    "type": "object",
+    "properties": {
+      "productId": {
+        "description": "The unique identifier for a product",
+        "type": "integer"
+      },
+      "productName": {
+        "description": "Name of the product",
+        "type": "string"
+      },
+      "price": {
+        "description": "The price of the product",
+        "type": "number",
+        "exclusiveMinimum": 0
+      },
+      "tags": {
+        "description": "Tags for the product",
+        "type": "array",
+        "items": {
+          "type": "string"
+        },
+        "minItems": 1,
+        "uniqueItems": true
+      },
+      "dimensions": {
+        "type": "object",
+        "properties": {
+          "length": {
+            "type": "number"
+          },
+          "width": {
+            "type": "number"
+          },
+          "height": {
+            "type": "number"
+          }
+        },
+        "required": [ "length", "width", "height" ]
+      }
+    },
+    "required": [ "productId", "productName", "price" ]
+  }
\ No newline at end of file
diff --git a/src/connector/src/test_data/proto_recursive/recursive.pb b/src/connector/src/test_data/proto_recursive/recursive.pb
index eb5e822e12956..5c611c18d0d30 100644
Binary files a/src/connector/src/test_data/proto_recursive/recursive.pb and b/src/connector/src/test_data/proto_recursive/recursive.pb differ
diff --git a/src/connector/src/test_data/proto_recursive/recursive.proto b/src/connector/src/test_data/proto_recursive/recursive.proto
index fe664c8d59cf6..93f177055788c 100644
--- a/src/connector/src/test_data/proto_recursive/recursive.proto
+++ b/src/connector/src/test_data/proto_recursive/recursive.proto
@@ -72,8 +72,8 @@ message AllTypes {
     EnumType oneof_enum = 21;
   }
 
-  // map field
-  map<string, int32> map_field = 22;
+  // // map field
+  // map<string, int32> map_field = 22;
 
   // timestamp
   google.protobuf.Timestamp timestamp_field = 23;
diff --git a/src/ctl/Cargo.toml b/src/ctl/Cargo.toml
index f271642ef4b76..cb713c92d2e6b 100644
--- a/src/ctl/Cargo.toml
+++ b/src/ctl/Cargo.toml
@@ -23,7 +23,7 @@ etcd-client = { workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 inquire = "0.6.2"
 itertools = "0.11"
-regex = "1.9.3"
+regex = "1.9.4"
 risingwave_common = { workspace = true }
 risingwave_connector = { workspace = true }
 risingwave_frontend = { workspace = true }
diff --git a/src/ctl/src/cmd_impl/bench.rs b/src/ctl/src/cmd_impl/bench.rs
index 3a0c6d5347a26..823febed50a43 100644
--- a/src/ctl/src/cmd_impl/bench.rs
+++ b/src/ctl/src/cmd_impl/bench.rs
@@ -103,7 +103,7 @@ pub async fn do_bench(context: &CtlContext, cmd: BenchCommands) -> Result<()> {
                     };
                     loop {
                         let stream = state_table
-                            .iter(PrefetchOptions::new_for_exhaust_iter())
+                            .iter_row(PrefetchOptions::new_for_exhaust_iter())
                             .await?;
                         pin_mut!(stream);
                         iter_cnt.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
diff --git a/src/ctl/src/cmd_impl/hummock.rs b/src/ctl/src/cmd_impl/hummock.rs
index 74886cb09737e..7f87c4f23ca21 100644
--- a/src/ctl/src/cmd_impl/hummock.rs
+++ b/src/ctl/src/cmd_impl/hummock.rs
@@ -23,9 +23,11 @@ mod list_version_deltas;
 mod pause_resume;
 mod trigger_full_gc;
 mod trigger_manual_compaction;
+mod validate_version;
 
 pub use compaction_group::*;
 pub use list_version_deltas::*;
 pub use pause_resume::*;
 pub use trigger_full_gc::*;
 pub use trigger_manual_compaction::*;
+pub use validate_version::*;
diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs
new file mode 100644
index 0000000000000..93407371746af
--- /dev/null
+++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs
@@ -0,0 +1,32 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext;
+use risingwave_rpc_client::HummockMetaClient;
+
+use crate::CtlContext;
+
+pub async fn validate_version(context: &CtlContext) -> anyhow::Result<()> {
+    let meta_client = context.meta_client().await?;
+    let version = meta_client.get_current_version().await?;
+    let result = hummock_version_ext::validate_version(&version);
+    if !result.is_empty() {
+        println!("Invalid HummockVersion. Violation lists:");
+        for s in result {
+            println!("{}", s);
+        }
+    }
+
+    Ok(())
+}
diff --git a/src/ctl/src/cmd_impl/meta/pause_resume.rs b/src/ctl/src/cmd_impl/meta/pause_resume.rs
index a75b653fa5bc4..5026d11b5b0e6 100644
--- a/src/ctl/src/cmd_impl/meta/pause_resume.rs
+++ b/src/ctl/src/cmd_impl/meta/pause_resume.rs
@@ -12,14 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_pb::meta::PausedReason;
+
 use crate::CtlContext;
 
+fn desc(reason: PausedReason) -> &'static str {
+    // Method on optional enums derived from `prost` will use `Unspecified` if unset. So we treat
+    // `Unspecified` as not paused here.
+    match reason {
+        PausedReason::Unspecified => "not paused",
+        PausedReason::ConfigChange => "paused due to configuration change",
+        PausedReason::Manual => "paused manually",
+    }
+}
+
 pub async fn pause(context: &CtlContext) -> anyhow::Result<()> {
     let meta_client = context.meta_client().await?;
 
-    meta_client.pause().await?;
+    let response = meta_client.pause().await?;
 
-    println!("Paused");
+    println!(
+        "Done.\nPrevious: {}\nCurrent: {}",
+        desc(response.prev()),
+        desc(response.curr())
+    );
 
     Ok(())
 }
@@ -27,9 +43,13 @@ pub async fn pause(context: &CtlContext) -> anyhow::Result<()> {
 pub async fn resume(context: &CtlContext) -> anyhow::Result<()> {
     let meta_client = context.meta_client().await?;
 
-    meta_client.resume().await?;
+    let response = meta_client.resume().await?;
 
-    println!("Resumed");
+    println!(
+        "Done.\nPrevious: {}\nCurrent: {}",
+        desc(response.prev()),
+        desc(response.curr())
+    );
 
     Ok(())
 }
diff --git a/src/ctl/src/cmd_impl/scale/resize.rs b/src/ctl/src/cmd_impl/scale/resize.rs
index 867109ee15878..46063c9075024 100644
--- a/src/ctl/src/cmd_impl/scale/resize.rs
+++ b/src/ctl/src/cmd_impl/scale/resize.rs
@@ -27,7 +27,7 @@ use serde_yaml;
 
 use crate::cmd_impl::meta::ReschedulePayload;
 use crate::common::CtlContext;
-use crate::ScaleResizeCommands;
+use crate::{ScaleCommon, ScaleHorizonCommands, ScaleVerticalCommands};
 
 macro_rules! fail {
     ($($arg:tt)*) => {{
@@ -36,8 +36,74 @@ macro_rules! fail {
     }};
 }
 
-pub async fn resize(context: &CtlContext, resize: ScaleResizeCommands) -> anyhow::Result<()> {
-    let meta_client = context.meta_client().await?;
+impl From<ScaleHorizonCommands> for ScaleCommandContext {
+    fn from(value: ScaleHorizonCommands) -> Self {
+        let ScaleHorizonCommands {
+            exclude_workers,
+            include_workers,
+            target_parallelism,
+            common:
+                ScaleCommon {
+                    generate,
+                    output,
+                    yes,
+                    fragments,
+                },
+        } = value;
+
+        Self {
+            exclude_workers,
+            include_workers,
+            target_parallelism,
+            generate,
+            output,
+            yes,
+            fragments,
+            target_parallelism_per_worker: None,
+        }
+    }
+}
+
+impl From<ScaleVerticalCommands> for ScaleCommandContext {
+    fn from(value: ScaleVerticalCommands) -> Self {
+        let ScaleVerticalCommands {
+            workers,
+            target_parallelism_per_worker,
+            common:
+                ScaleCommon {
+                    generate,
+                    output,
+                    yes,
+                    fragments,
+                },
+        } = value;
+
+        Self {
+            exclude_workers: None,
+            include_workers: workers,
+            target_parallelism: None,
+            generate,
+            output,
+            yes,
+            fragments,
+            target_parallelism_per_worker,
+        }
+    }
+}
+
+pub struct ScaleCommandContext {
+    exclude_workers: Option<Vec<String>>,
+    include_workers: Option<Vec<String>>,
+    target_parallelism: Option<u32>,
+    generate: bool,
+    output: Option<String>,
+    yes: bool,
+    fragments: Option<Vec<u32>>,
+    target_parallelism_per_worker: Option<u32>,
+}
+
+pub async fn resize(ctl_ctx: &CtlContext, scale_ctx: ScaleCommandContext) -> anyhow::Result<()> {
+    let meta_client = ctl_ctx.meta_client().await?;
 
     let GetClusterInfoResponse {
         worker_nodes,
@@ -116,15 +182,16 @@ pub async fn resize(context: &CtlContext, resize: ScaleResizeCommands) -> anyhow
         streaming_workers_index_by_id.len()
     );
 
-    let ScaleResizeCommands {
+    let ScaleCommandContext {
         exclude_workers,
         include_workers,
         target_parallelism,
+        target_parallelism_per_worker,
         generate,
         output,
         yes,
         fragments,
-    } = resize;
+    } = scale_ctx;
 
     let worker_changes = {
         let exclude_worker_ids =
@@ -132,8 +199,15 @@ pub async fn resize(context: &CtlContext, resize: ScaleResizeCommands) -> anyhow
         let include_worker_ids =
             worker_input_to_worker_ids(include_workers.unwrap_or_default(), true);
 
-        if let Some(target) = target_parallelism && target == 0 {
-            fail!("Target parallelism must be greater than 0");
+        match (target_parallelism, target_parallelism_per_worker) {
+            (Some(_), Some(_)) => {
+                fail!("Cannot specify both target parallelism and target parallelism per worker")
+            }
+            (_, Some(_)) if include_worker_ids.is_empty() => {
+                fail!("Cannot specify target parallelism per worker without including any worker")
+            }
+            (Some(target), _) if target == 0 => fail!("Target parallelism must be greater than 0"),
+            _ => {}
         }
 
         for worker_id in exclude_worker_ids.iter().chain(include_worker_ids.iter()) {
@@ -161,6 +235,7 @@ pub async fn resize(context: &CtlContext, resize: ScaleResizeCommands) -> anyhow
             include_worker_ids,
             exclude_worker_ids,
             target_parallelism,
+            target_parallelism_per_worker,
         }
     };
 
diff --git a/src/ctl/src/cmd_impl/table/scan.rs b/src/ctl/src/cmd_impl/table/scan.rs
index d110d024e467c..4c1ff68963613 100644
--- a/src/ctl/src/cmd_impl/table/scan.rs
+++ b/src/ctl/src/cmd_impl/table/scan.rs
@@ -129,7 +129,7 @@ async fn do_scan(table: TableCatalog, hummock: MonitoredStateStore<HummockStorag
         .await?;
     pin_mut!(stream);
     while let Some(item) = stream.next().await {
-        println!("{:?}", item?);
+        println!("{:?}", item?.into_owned_row());
     }
     Ok(())
 }
diff --git a/src/ctl/src/lib.rs b/src/ctl/src/lib.rs
index a576ffa43dbfb..adcb6cf3b3472 100644
--- a/src/ctl/src/lib.rs
+++ b/src/ctl/src/lib.rs
@@ -239,6 +239,8 @@ enum HummockCommands {
         #[clap(short, long = "verbose", default_value_t = false)]
         verbose: bool,
     },
+    /// Validate the current HummockVersion.
+    ValidateVersion,
 }
 
 #[derive(Subcommand)]
@@ -261,9 +263,8 @@ enum TableCommands {
     List,
 }
 
-#[derive(clap::Args, Debug)]
-#[clap(group(clap::ArgGroup::new("workers_group").required(true).multiple(true).args(&["include_workers", "exclude_workers", "target_parallelism"])))]
-pub struct ScaleResizeCommands {
+#[derive(clap::Args, Debug, Clone)]
+pub struct ScaleHorizonCommands {
     /// The worker that needs to be excluded during scheduling, worker_id and worker_host are both
     /// supported
     #[clap(
@@ -288,6 +289,12 @@ pub struct ScaleResizeCommands {
     #[clap(long)]
     target_parallelism: Option<u32>,
 
+    #[command(flatten)]
+    common: ScaleCommon,
+}
+
+#[derive(clap::Args, Debug, Clone)]
+pub struct ScaleCommon {
     /// Will generate a plan supported by the `reschedule` command and save it to the provided path
     /// by the `--output`.
     #[clap(long, default_value_t = false)]
@@ -307,12 +314,37 @@ pub struct ScaleResizeCommands {
     fragments: Option<Vec<u32>>,
 }
 
+#[derive(clap::Args, Debug, Clone)]
+pub struct ScaleVerticalCommands {
+    #[command(flatten)]
+    common: ScaleCommon,
+
+    /// The worker that needs to be scheduled, worker_id and worker_host are both
+    /// supported
+    #[clap(
+        long,
+        value_delimiter = ',',
+        value_name = "all or worker_id or worker_host, ..."
+    )]
+    workers: Option<Vec<String>>,
+
+    /// The target parallelism per worker, requires `workers` to be set.
+    #[clap(long, requires = "workers")]
+    target_parallelism_per_worker: Option<u32>,
+}
+
 #[derive(Subcommand, Debug)]
 enum ScaleCommands {
-    /// The resize command scales the cluster by specifying the workers to be included and
-    /// excluded.
-    Resize(ScaleResizeCommands),
-    /// mark a compute node as unschedulable
+    /// Scale the compute nodes horizontally, alias of `horizon`
+    Resize(ScaleHorizonCommands),
+
+    /// Scale the compute nodes horizontally
+    Horizon(ScaleHorizonCommands),
+
+    /// Scale the compute nodes vertically
+    Vertical(ScaleVerticalCommands),
+
+    /// Mark a compute node as unschedulable
     #[clap(verbatim_doc_comment)]
     Cordon {
         /// Workers that need to be cordoned, both id and host are supported.
@@ -354,12 +386,14 @@ enum MetaCommands {
     /// `added` when both are provided.
     ///
     /// For example, for plan `100-[1,2,3]+[4,5]` the follow request will be generated:
+    /// ```text
     /// {
     ///     100: Reschedule {
     ///         added_parallel_units: [4,5],
     ///         removed_parallel_units: [1,2,3],
     ///     }
     /// }
+    /// ```
     /// Use ; to separate multiple fragment
     #[clap(verbatim_doc_comment)]
     #[clap(group(clap::ArgGroup::new("input_group").required(true).args(&["plan", "from"])))]
@@ -556,6 +590,9 @@ pub async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
         Commands::Hummock(HummockCommands::ListCompactionStatus { verbose }) => {
             cmd_impl::hummock::list_compaction_status(context, verbose).await?;
         }
+        Commands::Hummock(HummockCommands::ValidateVersion) => {
+            cmd_impl::hummock::validate_version(context).await?;
+        }
         Commands::Table(TableCommands::Scan { mv_name, data_dir }) => {
             cmd_impl::table::scan(context, mv_name, data_dir).await?
         }
@@ -605,8 +642,12 @@ pub async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
         Commands::Profile(ProfileCommands::Heap { dir }) => {
             cmd_impl::profile::heap_profile(context, dir).await?
         }
-        Commands::Scale(ScaleCommands::Resize(resize)) => {
-            cmd_impl::scale::resize(context, resize).await?
+        Commands::Scale(ScaleCommands::Horizon(resize))
+        | Commands::Scale(ScaleCommands::Resize(resize)) => {
+            cmd_impl::scale::resize(context, resize.into()).await?
+        }
+        Commands::Scale(ScaleCommands::Vertical(resize)) => {
+            cmd_impl::scale::resize(context, resize.into()).await?
         }
         Commands::Scale(ScaleCommands::Cordon { workers }) => {
             cmd_impl::scale::update_schedulability(context, workers, Schedulability::Unschedulable)
diff --git a/src/expr/Cargo.toml b/src/expr/Cargo.toml
index e3928499dd685..7b3fccd17bffa 100644
--- a/src/expr/Cargo.toml
+++ b/src/expr/Cargo.toml
@@ -17,8 +17,8 @@ normal = ["workspace-hack", "ctor"]
 [dependencies]
 aho-corasick = "1"
 anyhow = "1"
-arrow-array = "45"
-arrow-schema = "45"
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
 async-trait = "0.1"
 auto_enums = "0.8"
 await-tree = { workspace = true }
diff --git a/src/expr/src/agg/def.rs b/src/expr/src/agg/def.rs
index d189bb5158092..570e910b68714 100644
--- a/src/expr/src/agg/def.rs
+++ b/src/expr/src/agg/def.rs
@@ -304,11 +304,11 @@ impl AggKind {
     }
 }
 
-/// Macros to generate match arms for [`AggKind`].
-/// IMPORTANT: These macros must be carefully maintained especially when adding new [`AggKind`]
-/// variants.
+/// Macros to generate match arms for [`AggKind`](crate::agg::AggKind).
+/// IMPORTANT: These macros must be carefully maintained especially when adding new
+/// [`AggKind`](crate::agg::AggKind) variants.
 pub mod agg_kinds {
-    /// [`AggKind`]s that are currently not supported in streaming mode.
+    /// [`AggKind`](crate::agg::AggKind)s that are currently not supported in streaming mode.
     #[macro_export]
     macro_rules! unimplemented_in_stream {
         () => {
@@ -325,8 +325,8 @@ pub mod agg_kinds {
     }
     pub use unimplemented_in_stream;
 
-    /// [`AggKind`]s that should've been rewritten to other kinds. These kinds should not appear
-    /// when generating physical plan nodes.
+    /// [`AggKind`](crate::agg::AggKind)s that should've been rewritten to other kinds. These kinds
+    /// should not appear when generating physical plan nodes.
     #[macro_export]
     macro_rules! rewritten {
         () => {
@@ -340,8 +340,8 @@ pub mod agg_kinds {
     }
     pub use rewritten;
 
-    /// [`AggKind`]s of which the aggregate results are not affected by the user given ORDER BY
-    /// clause.
+    /// [`AggKind`](crate::agg::AggKind)s of which the aggregate results are not affected by the
+    /// user given ORDER BY clause.
     #[macro_export]
     macro_rules! result_unaffected_by_order_by {
         () => {
@@ -365,9 +365,9 @@ pub mod agg_kinds {
     }
     pub use result_unaffected_by_order_by;
 
-    /// [`AggKind`]s that must be called with ORDER BY clause. These are slightly different from
-    /// variants not in [`result_unaffected_by_order_by`], in that variants returned by this macro
-    /// should be banned while the others should just be warned.
+    /// [`AggKind`](crate::agg::AggKind)s that must be called with ORDER BY clause. These are
+    /// slightly different from variants not in [`result_unaffected_by_order_by`], in that
+    /// variants returned by this macro should be banned while the others should just be warned.
     #[macro_export]
     macro_rules! must_have_order_by {
         () => {
@@ -380,8 +380,8 @@ pub mod agg_kinds {
     }
     pub use must_have_order_by;
 
-    /// [`AggKind`]s of which the aggregate results are not affected by the user given DISTINCT
-    /// keyword.
+    /// [`AggKind`](crate::agg::AggKind)s of which the aggregate results are not affected by the
+    /// user given DISTINCT keyword.
     #[macro_export]
     macro_rules! result_unaffected_by_distinct {
         () => {
@@ -396,7 +396,7 @@ pub mod agg_kinds {
     }
     pub use result_unaffected_by_distinct;
 
-    /// [`AggKind`]s that are simply cannot 2-phased.
+    /// [`AggKind`](crate::agg::AggKind)s that are simply cannot 2-phased.
     #[macro_export]
     macro_rules! simply_cannot_two_phase {
         () => {
@@ -414,7 +414,8 @@ pub mod agg_kinds {
     }
     pub use simply_cannot_two_phase;
 
-    /// [`AggKind`]s that are implemented with a single value state (so-called stateless).
+    /// [`AggKind`](crate::agg::AggKind)s that are implemented with a single value state (so-called
+    /// stateless).
     #[macro_export]
     macro_rules! single_value_state {
         () => {
@@ -427,8 +428,8 @@ pub mod agg_kinds {
     }
     pub use single_value_state;
 
-    /// [`AggKind`]s that are implemented with a single value state (so-called stateless) iff the
-    /// input is append-only.
+    /// [`AggKind`](crate::agg::AggKind)s that are implemented with a single value state (so-called
+    /// stateless) iff the input is append-only.
     #[macro_export]
     macro_rules! single_value_state_iff_in_append_only {
         () => {
diff --git a/src/expr/src/expr/build.rs b/src/expr/src/expr/build.rs
index 5b6a337578075..55ec16b4fe0fe 100644
--- a/src/expr/src/expr/build.rs
+++ b/src/expr/src/expr/build.rs
@@ -20,17 +20,19 @@ use risingwave_pb::expr::expr_node::{PbType, RexNode};
 use risingwave_pb::expr::ExprNode;
 
 use super::expr_array_concat::ArrayConcatExpression;
+use super::expr_array_transform::ArrayTransformExpression;
 use super::expr_case::CaseExpression;
 use super::expr_coalesce::CoalesceExpression;
 use super::expr_concat_ws::ConcatWsExpression;
 use super::expr_field::FieldExpression;
 use super::expr_in::InExpression;
 use super::expr_nested_construct::NestedConstructExpression;
-use super::expr_regexp::RegexpMatchExpression;
+use super::expr_regexp::{RegexpMatchExpression, RegexpReplaceExpression};
 use super::expr_some_all::SomeAllExpression;
 use super::expr_udf::UdfExpression;
 use super::expr_vnode::VnodeExpression;
 use crate::expr::expr_proctime::ProcTimeExpression;
+use crate::expr::expr_regexp_count::RegexpCountExpression;
 use crate::expr::{
     BoxedExpression, Expression, InputRefExpression, LiteralExpression, TryFromExprNodeBoxed,
 };
@@ -63,6 +65,8 @@ pub fn build_from_prost(prost: &ExprNode) -> Result<BoxedExpression> {
         E::Array => NestedConstructExpression::try_from_boxed(prost),
         E::Row => NestedConstructExpression::try_from_boxed(prost),
         E::RegexpMatch => RegexpMatchExpression::try_from_boxed(prost),
+        E::RegexpReplace => RegexpReplaceExpression::try_from_boxed(prost),
+        E::RegexpCount => RegexpCountExpression::try_from_boxed(prost),
         E::ArrayCat | E::ArrayAppend | E::ArrayPrepend => {
             // Now we implement these three functions as a single expression for the
             // sake of simplicity. If performance matters at some time, we can split
@@ -91,6 +95,12 @@ pub fn build_func(
     ret_type: DataType,
     children: Vec<BoxedExpression>,
 ) -> Result<BoxedExpression> {
+    if func == PbType::ArrayTransform {
+        // TODO: The function framework can't handle the lambda arg now.
+        let [array, lambda] = <[BoxedExpression; 2]>::try_from(children).unwrap();
+        return Ok(ArrayTransformExpression { array, lambda }.boxed());
+    }
+
     let args = children
         .iter()
         .map(|c| c.return_type().into())
diff --git a/src/expr/src/expr/expr_array_transform.rs b/src/expr/src/expr/expr_array_transform.rs
new file mode 100644
index 0000000000000..1f5d67397b332
--- /dev/null
+++ b/src/expr/src/expr/expr_array_transform.rs
@@ -0,0 +1,67 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use risingwave_common::array::{ArrayRef, DataChunk, Vis};
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{DataType, Datum, ListValue, ScalarImpl};
+
+use super::{BoxedExpression, Expression};
+use crate::Result;
+
+#[derive(Debug)]
+pub struct ArrayTransformExpression {
+    pub(super) array: BoxedExpression,
+    pub(super) lambda: BoxedExpression,
+}
+
+#[async_trait]
+impl Expression for ArrayTransformExpression {
+    fn return_type(&self) -> DataType {
+        DataType::List(Box::new(self.lambda.return_type()))
+    }
+
+    async fn eval(&self, input: &DataChunk) -> Result<ArrayRef> {
+        let lambda_input = self.array.eval_checked(input).await?;
+        let lambda_input = Arc::unwrap_or_clone(lambda_input).into_list();
+        let new_list = lambda_input
+            .map_inner(|flatten_input| async move {
+                let flatten_len = flatten_input.len();
+                let chunk =
+                    DataChunk::new(vec![Arc::new(flatten_input)], Vis::Compact(flatten_len));
+                self.lambda.eval(&chunk).await.map(Arc::unwrap_or_clone)
+            })
+            .await?;
+        Ok(Arc::new(new_list.into()))
+    }
+
+    async fn eval_row(&self, input: &OwnedRow) -> Result<Datum> {
+        let lambda_input = self.array.eval_row(input).await?;
+        let lambda_input = lambda_input.map(ScalarImpl::into_list);
+        if let Some(lambda_input) = lambda_input {
+            let mut new_vals = Vec::with_capacity(lambda_input.values().len());
+            for val in lambda_input.values() {
+                let row = OwnedRow::new(vec![val.clone()]);
+                let res = self.lambda.eval_row(&row).await?;
+                new_vals.push(res);
+            }
+            let new_list = ListValue::new(new_vals);
+            Ok(Some(new_list.into()))
+        } else {
+            Ok(None)
+        }
+    }
+}
diff --git a/src/expr/src/expr/expr_jsonb_access.rs b/src/expr/src/expr/expr_jsonb_access.rs
index 9516ad3a25bf1..5bcc76fd16705 100644
--- a/src/expr/src/expr/expr_jsonb_access.rs
+++ b/src/expr/src/expr/expr_jsonb_access.rs
@@ -25,7 +25,7 @@ use risingwave_expr_macro::build_function;
 use super::{BoxedExpression, Expression};
 use crate::Result;
 
-/// This is forked from [`BinaryExpression`] for the following reasons:
+/// This is forked from [`crate::expr::template::BinaryExpression`] for the following reasons:
 /// * Optimize for the case when rhs path is const. (not implemented yet)
 /// * It can return null when neither input is null.
 /// * We could `append(RefItem)` directly rather than getting a `OwnedItem` first.
diff --git a/src/expr/src/expr/expr_regexp.rs b/src/expr/src/expr/expr_regexp.rs
index 2c94146719af7..7907bb45ab915 100644
--- a/src/expr/src/expr/expr_regexp.rs
+++ b/src/expr/src/expr/expr_regexp.rs
@@ -18,7 +18,8 @@ use std::sync::Arc;
 use itertools::Itertools;
 use regex::{Regex, RegexBuilder};
 use risingwave_common::array::{
-    Array, ArrayBuilder, ArrayRef, DataChunk, ListArrayBuilder, ListRef, ListValue, Utf8Array,
+    Array, ArrayBuilder, ArrayImpl, ArrayRef, DataChunk, ListArrayBuilder, ListRef, ListValue,
+    Utf8Array, Utf8ArrayBuilder,
 };
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::{DataType, Datum, ScalarImpl};
@@ -31,16 +32,20 @@ use super::{build_from_prost as expr_build_from_prost, Expression};
 use crate::{bail, ensure, ExprError, Result};
 
 #[derive(Debug)]
-pub struct RegexpContext(pub Regex);
+pub struct RegexpContext {
+    pub regex: Regex,
+    pub global: bool,
+}
 
 impl RegexpContext {
     pub fn new(pattern: &str, flags: &str) -> Result<Self> {
         let options = RegexpOptions::from_str(flags)?;
-        Ok(Self(
-            RegexBuilder::new(pattern)
+        Ok(Self {
+            regex: RegexBuilder::new(pattern)
                 .case_insensitive(options.case_insensitive)
                 .build()?,
-        ))
+            global: options.global,
+        })
     }
 
     pub fn from_pattern(pattern: Datum) -> Result<Self> {
@@ -71,6 +76,8 @@ impl RegexpContext {
 struct RegexpOptions {
     /// `c` and `i`
     case_insensitive: bool,
+    /// `g`
+    global: bool,
 }
 
 #[expect(clippy::derivable_impls)]
@@ -78,6 +85,7 @@ impl Default for RegexpOptions {
     fn default() -> Self {
         Self {
             case_insensitive: false,
+            global: false,
         }
     }
 }
@@ -89,9 +97,12 @@ impl FromStr for RegexpOptions {
         let mut opts = Self::default();
         for c in s.chars() {
             match c {
+                // Case sensitive matching here
                 'c' => opts.case_insensitive = false,
+                // Case insensitive matching here
                 'i' => opts.case_insensitive = true,
-                'g' => {}
+                // Global matching here
+                'g' => opts.global = true,
                 _ => {
                     bail!("invalid regular expression option: \"{c}\"");
                 }
@@ -138,7 +149,7 @@ impl<'a> TryFrom<&'a ExprNode> for RegexpMatchExpression {
                     Some(ScalarImpl::Utf8(pattern)) => pattern.to_string(),
                     // NULL pattern
                     None => NULL_PATTERN.to_string(),
-                    _ => bail!("Expected pattern to be an String"),
+                    _ => bail!("Expected pattern to be a String"),
                 }
             }
             _ => {
@@ -164,7 +175,7 @@ impl<'a> TryFrom<&'a ExprNode> for RegexpMatchExpression {
                             pattern = NULL_PATTERN.to_string();
                             "".to_string()
                         }
-                        _ => bail!("Expected flags to be an String"),
+                        _ => bail!("Expected flags to be a String"),
                     }
                 }
                 _ => {
@@ -191,10 +202,10 @@ impl RegexpMatchExpression {
     fn match_one(&self, text: Option<&str>) -> Option<ListValue> {
         // If there are multiple captures, then the first one is the whole match, and should be
         // ignored in PostgreSQL's behavior.
-        let skip_flag = self.ctx.0.captures_len() > 1;
+        let skip_flag = self.ctx.regex.captures_len() > 1;
 
         if let Some(text) = text {
-            if let Some(capture) = self.ctx.0.captures(text) {
+            if let Some(capture) = self.ctx.regex.captures(text) {
                 let list = capture
                     .iter()
                     .skip(if skip_flag { 1 } else { 0 })
@@ -248,3 +259,498 @@ impl Expression for RegexpMatchExpression {
         })
     }
 }
+
+#[derive(Debug)]
+pub struct RegexpReplaceExpression {
+    /// The source to be matched and replaced
+    pub source: Box<dyn Expression>,
+    /// The regex context, used to match the given pattern
+    /// Contains `flag` relevant information, now we support `icg` flag options
+    pub ctx: RegexpContext,
+    /// The replacement string
+    pub replacement: String,
+    /// The actual return type by evaluating this expression
+    pub return_type: DataType,
+    /// The start position to replace the source
+    /// The starting index should be `0`
+    pub start: Option<u32>,
+    /// The N, used to specified the N-th position to be replaced
+    /// Note that this field is only available if `start` > 0
+    pub n: Option<u32>,
+}
+
+/// This trait provides the transformation from `ExprNode` to `RegexpReplaceExpression`
+impl<'a> TryFrom<&'a ExprNode> for RegexpReplaceExpression {
+    type Error = ExprError;
+
+    /// Try to convert the given `ExprNode` to the replace expression
+    fn try_from(prost: &'a ExprNode) -> Result<Self> {
+        // The function type must be of Type::RegexpReplace
+        ensure!(prost.get_function_type().unwrap() == Type::RegexpReplace);
+
+        // Get the return type first
+        let return_type = DataType::from(prost.get_return_type().unwrap());
+
+        // Get the top node, which must be the function call node in this case
+        let RexNode::FuncCall(func_call_node) = prost.get_rex_node().unwrap() else {
+            bail!("Expected RexNode::FuncCall");
+        };
+
+        // The children node, must contain `source`, `pattern`, `replacement`
+        // `start, N`, `flags` are optional
+        let mut children = func_call_node.children.iter();
+
+        // Get the source expression, will be used as the `child` in replace expr
+        let Some(source_node) = children.next() else {
+            bail!("Expected argument text");
+        };
+        let source = expr_build_from_prost(source_node)?;
+
+        // Get the regex pattern of this call
+        let Some(pattern_node) = children.next() else {
+            bail!("Expected argument pattern");
+        };
+        // Store the pattern as the string, to pass in the regex context
+        let pattern = match &pattern_node.get_rex_node()? {
+            RexNode::Constant(pattern_value) => {
+                let pattern_datum = deserialize_datum(
+                    pattern_value.get_body().as_slice(),
+                    &DataType::from(pattern_node.get_return_type().unwrap()),
+                )
+                .map_err(|e| ExprError::Internal(e.into()))?;
+
+                match pattern_datum {
+                    Some(ScalarImpl::Utf8(pattern)) => pattern.to_string(),
+                    // NULL pattern
+                    None => NULL_PATTERN.to_string(),
+                    _ => bail!("Expected pattern to be a String"),
+                }
+            }
+            _ => {
+                return Err(ExprError::UnsupportedFunction(
+                    "non-constant pattern in regexp_replace".to_string(),
+                ))
+            }
+        };
+
+        // Get the replacement string of this call
+        let Some(replacement_node) = children.next() else {
+            bail!("Expected argument replacement");
+        };
+        // Same as the pattern above, store as the string
+        let replacement = match &replacement_node.get_rex_node()? {
+            RexNode::Constant(replacement_value) => {
+                let replacement_datum = deserialize_datum(
+                    replacement_value.get_body().as_slice(),
+                    &DataType::from(replacement_node.get_return_type().unwrap()),
+                )
+                .map_err(|e| ExprError::Internal(e.into()))?;
+
+                match replacement_datum {
+                    Some(ScalarImpl::Utf8(replacement)) => replacement.to_string(),
+                    // NULL replacement
+                    // FIXME: Do we need the NULL match arm here?
+                    _ => bail!("Expected replacement to be a String"),
+                }
+            }
+            _ => {
+                return Err(ExprError::UnsupportedFunction(
+                    "non-constant in regexp_replace".to_string(),
+                ))
+            }
+        };
+
+        // The parsing for [, start [, N ]] [, flags ] options
+        let mut flags: Option<String> = None;
+        let mut start: Option<u32> = None;
+        let mut n: Option<u32> = None;
+        let mut n_flag = false;
+        let mut f_flag = false;
+
+        // Try to get the next possible node, see if any of the options are specified
+        if let Some(placeholder_node) = children.next() {
+            // Get the placeholder text first
+            let _placeholder = match &placeholder_node.get_rex_node()? {
+                RexNode::Constant(placeholder_value) => {
+                    let placeholder_datum = deserialize_datum(
+                        placeholder_value.get_body().as_slice(),
+                        &DataType::from(placeholder_node.get_return_type().unwrap()),
+                    )
+                    .map_err(|e| ExprError::Internal(e.into()))?;
+
+                    match placeholder_datum {
+                        Some(ScalarImpl::Int32(v)) => {
+                            if v <= 0 {
+                                // `start` must be greater than zero, if ever specified
+                                // This conforms with PG
+                                bail!("`start` must be greater than zero.");
+                            }
+                            start = Some(v as u32);
+                            "".to_string()
+                        }
+                        Some(ScalarImpl::Utf8(v)) => {
+                            // If the `start` is not specified
+                            // Then this must be the `flags`
+                            f_flag = true;
+                            flags = Some(v.to_string());
+                            "".to_string()
+                        }
+                        // NULL replacement
+                        // FIXME: Do we need the NULL match arm here?
+                        _ => bail!("Expected extra option to be a String/Int32"),
+                    }
+                }
+                _ => {
+                    return Err(ExprError::UnsupportedFunction(
+                        "non-constant in regexp_replace".to_string(),
+                    ))
+                }
+            };
+
+            // Get the next node
+            if !f_flag {
+                if let Some(placeholder_node) = children.next() {
+                    // Get the text as above
+                    let placeholder = match &placeholder_node.get_rex_node()? {
+                        RexNode::Constant(placeholder_value) => {
+                            let placeholder_datum = deserialize_datum(
+                                placeholder_value.get_body().as_slice(),
+                                &DataType::from(placeholder_node.get_return_type().unwrap()),
+                            )
+                            .map_err(|e| ExprError::Internal(e.into()))?;
+
+                            match placeholder_datum {
+                                Some(ScalarImpl::Int32(v)) => {
+                                    n_flag = true;
+                                    n = Some(v as u32);
+                                    "".to_string()
+                                }
+                                Some(ScalarImpl::Utf8(v)) => v.to_string(),
+                                // NULL replacement
+                                // FIXME: Do we need the NULL match arm here?
+                                _ => bail!("Expected extra option to be a String/Int32"),
+                            }
+                        }
+                        _ => {
+                            return Err(ExprError::UnsupportedFunction(
+                                "non-constant in regexp_replace".to_string(),
+                            ))
+                        }
+                    };
+
+                    if n_flag {
+                        // Check if any flag is specified
+                        if let Some(flag_node) = children.next() {
+                            // Get the flag
+                            flags = match &flag_node.get_rex_node()? {
+                                RexNode::Constant(flag_value) => {
+                                    let flag_datum = deserialize_datum(
+                                        flag_value.get_body().as_slice(),
+                                        &DataType::from(flag_node.get_return_type().unwrap()),
+                                    )
+                                    .map_err(|e| ExprError::Internal(e.into()))?;
+
+                                    match flag_datum {
+                                        Some(ScalarImpl::Utf8(v)) => Some(v.to_string()),
+                                        // NULL replacement
+                                        // FIXME: Do we need the NULL match arm here?
+                                        _ => bail!("Expected flag to be a String"),
+                                    }
+                                }
+                                _ => {
+                                    return Err(ExprError::UnsupportedFunction(
+                                        "non-constant in regexp_replace".to_string(),
+                                    ))
+                                }
+                            };
+                        }
+                    } else {
+                        flags = Some(placeholder);
+                    }
+                }
+            }
+        }
+
+        // TODO: Any other error handling?
+        if let Some(_other) = children.next() {
+            // There should not any other option after the `flags`
+            bail!("invalid parameters specified in regexp_replace");
+        }
+
+        // Check if the syntax is correct
+        if flags.is_some() && start.is_some() && n.is_none() {
+            // `start`, `flag` with no `N` specified is an invalid combination
+            bail!("invalid syntax for `regexp_replace`");
+        }
+
+        // Construct the final `RegexpReplaceExpression`
+        let flags = if let Some(f) = flags {
+            f
+        } else {
+            "".to_string()
+        };
+
+        // The `icg` flags will be set if ever specified
+        let ctx = RegexpContext::new(&pattern, &flags)?;
+
+        // Construct the regex used to match and replace `\n` expression
+        // Check: https://docs.rs/regex/latest/regex/struct.Captures.html#method.expand
+        let regex = Regex::new(r"\\([1-9])").unwrap();
+
+        // Get the replaced string
+        let replacement = regex
+            .replace_all(&replacement, "$${$1}")
+            // This is for the '\$' substitution
+            .replace("\\&", "${0}");
+
+        Ok(Self {
+            source,
+            ctx,
+            replacement,
+            return_type,
+            start,
+            n,
+        })
+    }
+}
+
+impl RegexpReplaceExpression {
+    /// Match and replace one row, return the replaced string
+    fn match_row(&self, text: Option<&str>) -> Option<String> {
+        if let Some(text) = text {
+            // The start position to begin the search
+            let start = if let Some(s) = self.start { s - 1 } else { 0 };
+
+            // This is because the source text may contain unicode
+            let start = match text.char_indices().nth(start as usize) {
+                Some((idx, _)) => idx,
+                // With no match
+                None => return Some(text.into()),
+            };
+
+            if (self.n.is_none() && self.ctx.global) || (self.n.is_some() && self.n.unwrap() == 0) {
+                // --------------------------------------------------------------
+                // `-g` enabled (& `N` is not specified) or `N` is `0`          |
+                // We need to replace all the occurrence of the matched pattern |
+                // --------------------------------------------------------------
+
+                // See if there is capture group or not
+                if self.ctx.regex.captures_len() <= 1 {
+                    // There is no capture groups in the regex
+                    // Just replace all matched patterns after `start`
+                    return Some(
+                        text[..start].to_string()
+                            + &self
+                                .ctx
+                                .regex
+                                .replace_all(&text[start..], self.replacement.clone()),
+                    );
+                } else {
+                    // The position to start searching for replacement
+                    let mut search_start = start;
+
+                    // Construct the return string
+                    let mut ret = text[..search_start].to_string();
+
+                    // Begin the actual replace logic
+                    while let Some(capture) = self.ctx.regex.captures(&text[search_start..]) {
+                        let match_start = capture.get(0).unwrap().start();
+                        let match_end = capture.get(0).unwrap().end();
+
+                        if match_start == match_end {
+                            // If this is an empty match
+                            search_start += 1;
+                            continue;
+                        }
+
+                        // Append the portion of the text from `search_start` to `match_start`
+                        ret.push_str(&text[search_start..search_start + match_start]);
+
+                        // Start to replacing
+                        // Note that the result will be written directly to `ret` buffer
+                        capture.expand(&self.replacement, &mut ret);
+
+                        // Update the `search_start`
+                        search_start += match_end;
+                    }
+
+                    // Push the rest of the text to return string
+                    ret.push_str(&text[search_start..]);
+
+                    Some(ret)
+                }
+            } else {
+                // -------------------------------------------------
+                // Only replace the first matched pattern          |
+                // Or the N-th matched pattern if `N` is specified |
+                // -------------------------------------------------
+
+                // Construct the return string
+                let mut ret = if start > 1 {
+                    text[..start].to_string()
+                } else {
+                    "".to_string()
+                };
+
+                // See if there is capture group or not
+                if self.ctx.regex.captures_len() <= 1 {
+                    // There is no capture groups in the regex
+                    if self.n.is_none() {
+                        // `N` is not specified
+                        ret.push_str(&self.ctx.regex.replacen(
+                            &text[start..],
+                            1,
+                            &self.replacement,
+                        ));
+                    } else {
+                        // Replace only the N-th match
+                        let mut count = 1;
+                        // The absolute index for the start of searching
+                        let mut search_start = start;
+                        while let Some(capture) = self.ctx.regex.captures(&text[search_start..]) {
+                            // Get the current start & end index
+                            let match_start = capture.get(0).unwrap().start();
+                            let match_end = capture.get(0).unwrap().end();
+
+                            if count == self.n.unwrap() as i32 {
+                                // We've reached the pattern to replace
+                                // Let's construct the return string
+                                ret = format!(
+                                    "{}{}{}",
+                                    &text[..search_start + match_start],
+                                    &self.replacement,
+                                    &text[search_start + match_end..]
+                                );
+                                break;
+                            }
+
+                            // Update the counter
+                            count += 1;
+
+                            // Update `start`
+                            search_start += match_end;
+                        }
+                    }
+                } else {
+                    // There are capture groups in the regex
+                    // Reset return string at the beginning
+                    ret = "".to_string();
+                    if self.n.is_none() {
+                        // `N` is not specified
+                        if self.ctx.regex.captures(&text[start..]).is_none() {
+                            // No match
+                            return Some(text.into());
+                        }
+                        // Otherwise replace the source text
+                        if let Some(capture) = self.ctx.regex.captures(&text[start..]) {
+                            let match_start = capture.get(0).unwrap().start();
+                            let match_end = capture.get(0).unwrap().end();
+
+                            // Get the replaced string and expand it
+                            capture.expand(&self.replacement, &mut ret);
+
+                            // Construct the return string
+                            ret = format!(
+                                "{}{}{}",
+                                &text[..start + match_start],
+                                ret,
+                                &text[start + match_end..]
+                            );
+                        }
+                    } else {
+                        // Replace only the N-th match
+                        let mut count = 1;
+                        while let Some(capture) = self.ctx.regex.captures(&text[start..]) {
+                            if count == self.n.unwrap() as i32 {
+                                // We've reached the pattern to replace
+                                let match_start = capture.get(0).unwrap().start();
+                                let match_end = capture.get(0).unwrap().end();
+
+                                // Get the replaced string and expand it
+                                capture.expand(&self.replacement, &mut ret);
+
+                                // Construct the return string
+                                ret = format!(
+                                    "{}{}{}",
+                                    &text[..start + match_start],
+                                    ret,
+                                    &text[start + match_end..]
+                                );
+                            }
+
+                            // Update the counter
+                            count += 1;
+                        }
+
+                        // If there is no match, just return the original string
+                        if ret.is_empty() {
+                            ret = text.into();
+                        }
+                    }
+                }
+
+                Some(ret)
+            }
+        } else {
+            // The input string is None
+            // Directly return
+            None
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Expression for RegexpReplaceExpression {
+    fn return_type(&self) -> DataType {
+        self.return_type.clone()
+    }
+
+    async fn eval(&self, input: &DataChunk) -> Result<ArrayRef> {
+        // Get the source text column first
+        let source_column = self.source.eval_checked(input).await?;
+        let source_column = source_column.as_utf8();
+
+        let row_len = input.capacity();
+        let vis = input.vis();
+        let mut builder = Utf8ArrayBuilder::new(row_len);
+
+        for row_idx in 0..row_len {
+            // If not visible, just append the `None`
+            if !vis.is_set(row_idx) {
+                builder.append(None);
+                continue;
+            }
+
+            // Try to get the source text for this column
+            let source = match source_column.value_at(row_idx) {
+                Some(s) => s,
+                None => {
+                    builder.append(None);
+                    continue;
+                }
+            };
+
+            if let Some(ret) = self.match_row(Some(source)) {
+                builder.append(Some(&ret));
+            } else {
+                builder.append(None);
+            }
+        }
+
+        Ok(Arc::new(ArrayImpl::from(builder.finish())))
+    }
+
+    async fn eval_row(&self, input: &OwnedRow) -> Result<Datum> {
+        // Get the source text to match and replace
+        let source = self.source.eval_row(input).await?;
+        let source = match source {
+            Some(ScalarImpl::Utf8(s)) => s,
+            // The input source is invalid, directly return None
+            _ => return Ok(None),
+        };
+
+        Ok(self
+            .match_row(Some(&source))
+            .map(|replaced| replaced.into()))
+    }
+}
diff --git a/src/expr/src/expr/expr_regexp_count.rs b/src/expr/src/expr/expr_regexp_count.rs
new file mode 100644
index 0000000000000..28e5b75ff74e9
--- /dev/null
+++ b/src/expr/src/expr/expr_regexp_count.rs
@@ -0,0 +1,229 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use risingwave_common::array::{
+    Array, ArrayBuilder, ArrayImpl, ArrayRef, DataChunk, I32ArrayBuilder,
+};
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{DataType, Datum, ScalarImpl};
+use risingwave_common::util::value_encoding::deserialize_datum;
+use risingwave_common::{bail, ensure};
+use risingwave_pb::expr::expr_node::{RexNode, Type};
+use risingwave_pb::expr::ExprNode;
+
+use super::expr_regexp::RegexpContext;
+use super::{build_from_prost as expr_build_from_prost, Expression};
+use crate::{ExprError, Result};
+
+#[derive(Debug)]
+pub struct RegexpCountExpression {
+    /// The source text
+    pub source: Box<dyn Expression>,
+    /// Relevant regex context, contains `flags` option
+    pub ctx: RegexpContext,
+    /// The start position to begin the counting process
+    pub start: Option<u32>,
+}
+
+pub const NULL_PATTERN: &str = "a^";
+
+/// This trait provides the transformation from `ExprNode` to `RegexpCountExpression`
+impl<'a> TryFrom<&'a ExprNode> for RegexpCountExpression {
+    type Error = ExprError;
+
+    fn try_from(prost: &'a ExprNode) -> Result<Self> {
+        // Sanity check first
+        ensure!(prost.get_function_type().unwrap() == Type::RegexpCount);
+
+        let RexNode::FuncCall(func_call_node) = prost.get_rex_node().unwrap() else {
+            bail!("Expected RexNode::FuncCall");
+        };
+
+        let mut children = func_call_node.children.iter();
+
+        let Some(source_node) = children.next() else {
+            bail!("Expected source text");
+        };
+        let source = expr_build_from_prost(source_node)?;
+
+        let Some(pattern_node) = children.next() else {
+            bail!("Expected pattern text");
+        };
+        let pattern = match &pattern_node.get_rex_node()? {
+            RexNode::Constant(pattern_value) => {
+                let pattern_datum = deserialize_datum(
+                    pattern_value.get_body().as_slice(),
+                    &DataType::from(pattern_node.get_return_type().unwrap()),
+                )
+                .map_err(|e| ExprError::Internal(e.into()))?;
+
+                match pattern_datum {
+                    Some(ScalarImpl::Utf8(pattern)) => pattern.to_string(),
+                    // NULL pattern
+                    None => NULL_PATTERN.to_string(),
+                    _ => bail!("Expected pattern to be a String"),
+                }
+            }
+            _ => {
+                return Err(ExprError::UnsupportedFunction(
+                    "non-constant pattern in `regexp_count`".to_string(),
+                ))
+            }
+        };
+
+        // Parsing for [ , start [, flags ]]
+        let mut flags: Option<String> = None;
+        let mut start: Option<u32> = None;
+
+        // See if `start` is specified
+        if let Some(start_node) = children.next() {
+            start = match &start_node.get_rex_node()? {
+                RexNode::Constant(start_value) => {
+                    let start_datum = deserialize_datum(
+                        start_value.get_body().as_slice(),
+                        &DataType::from(start_node.get_return_type().unwrap()),
+                    )
+                    .map_err(|e| ExprError::Internal(e.into()))?;
+
+                    match start_datum {
+                        Some(ScalarImpl::Int32(start)) => {
+                            if start <= 0 {
+                                bail!("start must greater than zero");
+                            }
+                            Some(start as u32)
+                        }
+                        _ => bail!("Expected start to be a Unsigned Int32"),
+                    }
+                }
+                _ => {
+                    return Err(ExprError::UnsupportedFunction(
+                        "non-constant start in `regexp_count`".to_string(),
+                    ))
+                }
+            };
+
+            // See if `flags` is specified
+            if let Some(flags_node) = children.next() {
+                flags = match &flags_node.get_rex_node()? {
+                    RexNode::Constant(flags_value) => {
+                        let flags_datum = deserialize_datum(
+                            flags_value.get_body().as_slice(),
+                            &DataType::from(flags_node.get_return_type().unwrap()),
+                        )
+                        .map_err(|e| ExprError::Internal(e.into()))?;
+
+                        match flags_datum {
+                            Some(ScalarImpl::Utf8(flags)) => Some(flags.to_string()),
+                            _ => bail!("Expected flags to be a String"),
+                        }
+                    }
+                    _ => {
+                        return Err(ExprError::UnsupportedFunction(
+                            "non-constant flags in `regexp_count`".to_string(),
+                        ))
+                    }
+                }
+            }
+        };
+
+        // Sanity check
+        if children.next().is_some() {
+            bail!("syntax error in `regexp_count`");
+        }
+
+        let flags = flags.unwrap_or_default();
+
+        if flags.contains('g') {
+            bail!("`regexp_count` does not support global flag option");
+        }
+
+        let ctx = RegexpContext::new(&pattern, &flags)?;
+
+        Ok(Self { source, ctx, start })
+    }
+}
+
+impl RegexpCountExpression {
+    fn match_row(&self, text: Option<&str>) -> Option<i32> {
+        if let Some(text) = text {
+            // First get the start position to count for
+            let start = if let Some(s) = self.start { s - 1 } else { 0 };
+
+            // For unicode purpose
+            let mut start = match text.char_indices().nth(start as usize) {
+                Some((idx, _)) => idx,
+                // The `start` is out of bound
+                None => return Some(0),
+            };
+
+            let mut count = 0;
+
+            while let Some(captures) = self.ctx.regex.captures(&text[start..]) {
+                count += 1;
+                start += captures.get(0).unwrap().end();
+            }
+
+            Some(count)
+        } else {
+            // Input string is None, the return value should be NULL
+            None
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Expression for RegexpCountExpression {
+    fn return_type(&self) -> DataType {
+        DataType::Int32
+    }
+
+    async fn eval(&self, input: &DataChunk) -> Result<ArrayRef> {
+        let source_column = self.source.eval_checked(input).await?;
+        let source_column = source_column.as_utf8();
+
+        let row_len = input.capacity();
+        let vis = input.vis();
+        let mut builder: I32ArrayBuilder = ArrayBuilder::new(row_len);
+
+        for row_idx in 0..row_len {
+            if !vis.is_set(row_idx) {
+                builder.append(None);
+                continue;
+            }
+
+            let source = source_column.value_at(row_idx);
+            builder.append(self.match_row(source));
+        }
+
+        Ok(Arc::new(ArrayImpl::from(builder.finish())))
+    }
+
+    async fn eval_row(&self, input: &OwnedRow) -> Result<Datum> {
+        let source = self.source.eval_row(input).await?;
+        // Will panic if the input text is not a String
+        let source = match source {
+            Some(ScalarImpl::Utf8(s)) => s,
+            None => return Ok(None),
+            // Other than the above cases
+            // The input is invalid and we should panic here
+            _ => bail!("source should be a String"),
+        };
+
+        Ok(self
+            .match_row(Some(&source))
+            .map(|replaced| replaced.into()))
+    }
+}
diff --git a/src/expr/src/expr/mod.rs b/src/expr/src/expr/mod.rs
index 496ff2e66d2b1..60f2484bf010d 100644
--- a/src/expr/src/expr/mod.rs
+++ b/src/expr/src/expr/mod.rs
@@ -18,7 +18,8 @@
 //!
 //! ## Construction
 //!
-//! Expressions can be constructed by [`build()`] function, which returns a [`BoxedExpression`].
+//! Expressions can be constructed by [`build_func()`] function, which returns a
+//! [`BoxedExpression`].
 //!
 //! They can also be transformed from the prost [`ExprNode`] using the [`build_from_prost()`]
 //! function.
@@ -33,6 +34,7 @@
 // These modules define concrete expression structures.
 mod expr_array_concat;
 mod expr_array_to_string;
+mod expr_array_transform;
 mod expr_binary_nonnull;
 mod expr_binary_nullable;
 mod expr_case;
@@ -47,6 +49,7 @@ mod expr_literal;
 mod expr_nested_construct;
 mod expr_proctime;
 pub mod expr_regexp;
+pub mod expr_regexp_count;
 mod expr_some_all;
 mod expr_timestamp_to_char_const_tmpl;
 mod expr_timestamptz_to_char_const_tmpl;
diff --git a/src/expr/src/lib.rs b/src/expr/src/lib.rs
index 4211f6182ab6c..7f955249909b2 100644
--- a/src/expr/src/lib.rs
+++ b/src/expr/src/lib.rs
@@ -23,6 +23,7 @@
 #![feature(round_ties_even)]
 #![feature(generators)]
 #![feature(test)]
+#![feature(arc_unwrap_or_clone)]
 
 pub mod agg;
 mod error;
diff --git a/src/expr/src/sig/func.rs b/src/expr/src/sig/func.rs
index d84a065ad095a..5dca4da2f4486 100644
--- a/src/expr/src/sig/func.rs
+++ b/src/expr/src/sig/func.rs
@@ -199,6 +199,12 @@ mod tests {
                 ArrayAccess: [
                     "array_access(list, int32) -> boolean/int16/int32/int64/int256/float32/float64/decimal/serial/date/time/timestamp/timestamptz/interval/varchar/bytea/jsonb/list/struct",
                 ],
+                ArrayMin: [
+                    "array_min(list) -> bytea/varchar/timestamptz/timestamp/time/date/int256/serial/decimal/float32/float64/int16/int32/int64",
+                ],
+                ArrayMax: [
+                    "array_max(list) -> bytea/varchar/timestamptz/timestamp/time/date/int256/serial/decimal/float32/float64/int16/int32/int64",
+                ],
             }
         "#]];
         expected.assert_debug_eq(&duplicated);
diff --git a/src/expr/src/table_function/regexp_matches.rs b/src/expr/src/table_function/regexp_matches.rs
index b8d5712f3de1e..961bbf2adb6e7 100644
--- a/src/expr/src/table_function/regexp_matches.rs
+++ b/src/expr/src/table_function/regexp_matches.rs
@@ -31,10 +31,10 @@ fn regexp_matches<'a>(
     text: &'a str,
     regex: &'a RegexpContext,
 ) -> impl Iterator<Item = ListValue> + 'a {
-    regex.0.captures_iter(text).map(|capture| {
+    regex.regex.captures_iter(text).map(|capture| {
         // If there are multiple captures, then the first one is the whole match, and should be
         // ignored in PostgreSQL's behavior.
-        let skip_flag = regex.0.captures_len() > 1;
+        let skip_flag = regex.regex.captures_len() > 1;
         let list = capture
             .iter()
             .skip(if skip_flag { 1 } else { 0 })
diff --git a/src/expr/src/vector_op/array_access.rs b/src/expr/src/vector_op/array_access.rs
index 57fe3f29feb3f..40c4568c7d467 100644
--- a/src/expr/src/vector_op/array_access.rs
+++ b/src/expr/src/vector_op/array_access.rs
@@ -24,7 +24,8 @@ pub fn array_access<T: Scalar>(list: ListRef<'_>, index: i32) -> Result<Option<T
     if index < 1 {
         return Ok(None);
     }
-    let datum_ref = list.elem_at(index as usize - 1).flatten(); // returns `NULL` if index is out of bounds
+    // returns `NULL` if index is out of bounds
+    let datum_ref = list.elem_at(index as usize - 1).flatten();
     if let Some(scalar) = datum_ref.to_owned_datum() {
         Ok(Some(scalar.try_into()?))
     } else {
diff --git a/src/expr/src/vector_op/array_distinct.rs b/src/expr/src/vector_op/array_distinct.rs
index dff8bc68eaa6d..22084b0e11ac7 100644
--- a/src/expr/src/vector_op/array_distinct.rs
+++ b/src/expr/src/vector_op/array_distinct.rs
@@ -20,7 +20,7 @@ use risingwave_expr_macro::function;
 /// Returns a new array removing all the duplicates from the input array
 ///
 /// ```sql
-/// array_distinct ( array anyarray) → array
+/// array_distinct (array anyarray) → array
 /// ```
 ///
 /// Examples:
diff --git a/src/expr/src/vector_op/array_length.rs b/src/expr/src/vector_op/array_length.rs
index 4b44b2ac4d9b4..e0f7267403196 100644
--- a/src/expr/src/vector_op/array_length.rs
+++ b/src/expr/src/vector_op/array_length.rs
@@ -20,7 +20,7 @@ use crate::ExprError;
 /// Returns the length of an array.
 ///
 /// ```sql
-/// array_length ( array anyarray) → int64
+/// array_length (array anyarray) → int64
 /// ```
 ///
 /// Examples:
diff --git a/src/expr/src/vector_op/array_min_max.rs b/src/expr/src/vector_op/array_min_max.rs
new file mode 100644
index 0000000000000..1ff1a4086f2cd
--- /dev/null
+++ b/src/expr/src/vector_op/array_min_max.rs
@@ -0,0 +1,59 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::array::*;
+use risingwave_common::types::{DefaultOrdered, Scalar, ToOwnedDatum};
+use risingwave_expr_macro::function;
+
+use crate::Result;
+
+/// FIXME: #[`function("array_min(list`) -> any")] supports
+/// In this way we could avoid manual macro expansion
+#[function("array_min(list) -> *int")]
+#[function("array_min(list) -> *float")]
+#[function("array_min(list) -> decimal")]
+#[function("array_min(list) -> serial")]
+#[function("array_min(list) -> int256")]
+#[function("array_min(list) -> date")]
+#[function("array_min(list) -> time")]
+#[function("array_min(list) -> timestamp")]
+#[function("array_min(list) -> timestamptz")]
+#[function("array_min(list) -> varchar")]
+#[function("array_min(list) -> bytea")]
+pub fn array_min<T: Scalar>(list: ListRef<'_>) -> Result<Option<T>> {
+    let min_value = list.iter().flatten().map(DefaultOrdered).min();
+    match min_value.map(|v| v.0).to_owned_datum() {
+        Some(s) => Ok(Some(s.try_into()?)),
+        None => Ok(None),
+    }
+}
+
+#[function("array_max(list) -> *int")]
+#[function("array_max(list) -> *float")]
+#[function("array_max(list) -> decimal")]
+#[function("array_max(list) -> serial")]
+#[function("array_max(list) -> int256")]
+#[function("array_max(list) -> date")]
+#[function("array_max(list) -> time")]
+#[function("array_max(list) -> timestamp")]
+#[function("array_max(list) -> timestamptz")]
+#[function("array_max(list) -> varchar")]
+#[function("array_max(list) -> bytea")]
+pub fn array_max<T: Scalar>(list: ListRef<'_>) -> Result<Option<T>> {
+    let max_value = list.iter().flatten().map(DefaultOrdered).max();
+    match max_value.map(|v| v.0).to_owned_datum() {
+        Some(s) => Ok(Some(s.try_into()?)),
+        None => Ok(None),
+    }
+}
diff --git a/src/expr/src/vector_op/mod.rs b/src/expr/src/vector_op/mod.rs
index 4bc147cf3caec..f4aaa375f8f76 100644
--- a/src/expr/src/vector_op/mod.rs
+++ b/src/expr/src/vector_op/mod.rs
@@ -16,6 +16,7 @@ pub mod arithmetic_op;
 pub mod array_access;
 pub mod array_distinct;
 pub mod array_length;
+pub mod array_min_max;
 pub mod array_positions;
 pub mod array_range_access;
 pub mod array_remove;
diff --git a/src/expr/src/vector_op/timestamptz.rs b/src/expr/src/vector_op/timestamptz.rs
index ca24200300244..716a521f742e4 100644
--- a/src/expr/src/vector_op/timestamptz.rs
+++ b/src/expr/src/vector_op/timestamptz.rs
@@ -276,17 +276,13 @@ mod tests {
 
     #[test]
     fn test_timestamptz_to_and_from_string() {
-        let str1 = "0001-11-15 15:35:40.999999+08:00";
+        let str1 = "1600-11-15 15:35:40.999999+08:00";
         let timestamptz1 = str_to_timestamptz(str1, "UTC").unwrap();
-        assert_eq!(timestamptz1.timestamp_micros(), -62108094259000001);
+        assert_eq!(timestamptz1.timestamp_micros(), -11648507059000001);
 
         let mut writer = String::new();
         timestamptz_to_string(timestamptz1, "UTC", &mut writer).unwrap();
-        assert_eq!(writer, "0001-11-15 07:35:40.999999+00:00");
-
-        let mut writer = String::new();
-        timestamptz_to_string(timestamptz1, "UTC", &mut writer).unwrap();
-        assert_eq!(writer, "0001-11-15 07:35:40.999999+00:00");
+        assert_eq!(writer, "1600-11-15 07:35:40.999999+00:00");
 
         let str2 = "1969-12-31 23:59:59.999999+00:00";
         let timestamptz2 = str_to_timestamptz(str2, "UTC").unwrap();
diff --git a/src/expr/src/window_function/call.rs b/src/expr/src/window_function/call.rs
index 19bb5a58d2e75..11586c6252bcc 100644
--- a/src/expr/src/window_function/call.rs
+++ b/src/expr/src/window_function/call.rs
@@ -78,6 +78,10 @@ impl Frame {
             exclusion,
         }
     }
+
+    pub fn is_unbounded(&self) -> bool {
+        self.bounds.is_unbounded()
+    }
 }
 
 impl Frame {
@@ -127,6 +131,10 @@ impl FrameBounds {
             Self::Rows(_, end) => matches!(end, FrameBound::UnboundedFollowing),
         }
     }
+
+    pub fn is_unbounded(&self) -> bool {
+        self.start_is_unbounded() || self.end_is_unbounded()
+    }
 }
 
 impl Display for FrameBounds {
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index 359230c0e47c5..a728f5999b37c 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -17,8 +17,8 @@ normal = ["workspace-hack"]
 [dependencies]
 anyhow = "1"
 arc-swap = "1"
-arrow-schema = "45"
-async-recursion = "1.0.4"
+arrow-schema = { workspace = true }
+async-recursion = "1.0.5"
 async-trait = "0.1"
 auto_enums = { version = "0.8", features = ["futures03"] }
 bk-tree = "0.5.0"
diff --git a/src/frontend/planner_test/src/lib.rs b/src/frontend/planner_test/src/lib.rs
index f31aa1443f74d..048851ebd17da 100644
--- a/src/frontend/planner_test/src/lib.rs
+++ b/src/frontend/planner_test/src/lib.rs
@@ -779,6 +779,8 @@ impl TestCase {
                     format!("CREATE SINK {sink_name} AS {}", stmt),
                     options,
                     false,
+                    "test_db".into(),
+                    "test_table".into(),
                 ) {
                     Ok(sink_plan) => {
                         ret.sink_plan = Some(explain_plan(&sink_plan.into()));
diff --git a/src/frontend/planner_test/tests/testdata/input/grouping_sets.yaml b/src/frontend/planner_test/tests/testdata/input/grouping_sets.yaml
index 53b016d6bb44a..08ad5fe160100 100644
--- a/src/frontend/planner_test/tests/testdata/input/grouping_sets.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/grouping_sets.yaml
@@ -73,3 +73,10 @@
   expected_outputs:
     - batch_plan
     - stream_plan
+- name: only one set in grouping sets
+  sql: |
+    create table items_sold (brand varchar, size varchar, sales int);
+    SELECT brand, sum(sales) FROM items_sold GROUP BY GROUPING SETS(brand);
+  expected_outputs:
+    - batch_plan
+    - stream_plan
diff --git a/src/frontend/planner_test/tests/testdata/input/project_set.yaml b/src/frontend/planner_test/tests/testdata/input/project_set.yaml
index 05da6311a3d18..96b8b072aa6c7 100644
--- a/src/frontend/planner_test/tests/testdata/input/project_set.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/project_set.yaml
@@ -93,3 +93,33 @@
   expected_outputs:
   - batch_plan
   - stream_plan
+- name: set-returning function disallowed in CASE WHEN
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select case when a1 is not null then unnest(a1) end from a;
+  expected_outputs:
+  - binder_error
+- name: set-returning function disallowed in COALESCE
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select coalesce(unnest(a1), 0) from a;
+  expected_outputs:
+  - binder_error
+- name: set-returning function disallowed in JOIN ON
+  sql: |
+    with a(a1) as (values (array[2]), (null)), b(b1) as (select 2) select a1 from a join b on unnest(a1) = b1;
+  expected_outputs:
+  - binder_error
+- name: set-returning function disallowed in HAVING
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select count(*) from a group by a1 having unnest(a1) < 5;
+  expected_outputs:
+  - binder_error
+- name: set-returning function disallowed in FILTER
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select count(*) filter(where unnest(a1) < 3) from a;
+  expected_outputs:
+  - binder_error
+- name: set-returning function unsupported in GROUP BY yet
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select count(*) from a group by unnest(a1);
+  expected_outputs:
+  - planner_error
diff --git a/src/frontend/planner_test/tests/testdata/input/watermark.yaml b/src/frontend/planner_test/tests/testdata/input/watermark.yaml
index c56045f8ed44c..1dfe8155b7817 100644
--- a/src/frontend/planner_test/tests/testdata/input/watermark.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/watermark.yaml
@@ -96,3 +96,9 @@
     select window_start from hop(t, ts, interval '1' minute, interval '3' minute);
   expected_outputs:
   - stream_plan
+- name: unnest
+  sql: |
+    create table t (ts timestamp with time zone, v1 int, watermark for ts as ts - INTERVAL '1' SECOND) append only;
+    explain create materialized view mv as select t.ts, unnest(Array[1,2,3]) from t emit on window close;
+  expected_outputs:
+    - explain_output
diff --git a/src/frontend/planner_test/tests/testdata/output/agg.yaml b/src/frontend/planner_test/tests/testdata/output/agg.yaml
index 6390c2f6ab331..fd57f0dfe9fdb 100644
--- a/src/frontend/planner_test/tests/testdata/output/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/agg.yaml
@@ -1099,8 +1099,7 @@
     Bind error: failed to bind expression: count(a1) FILTER(WHERE unnest(ARRAY[1]) < 1)
 
     Caused by:
-      Feature is not yet implemented: table function in filter clause
-    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+      Invalid input syntax: table functions are not allowed in FILTER
 - name: agg order by - subquery
   sql: |
     /* This case is valid in PostgreSQL */
diff --git a/src/frontend/planner_test/tests/testdata/output/create_source.yaml b/src/frontend/planner_test/tests/testdata/output/create_source.yaml
index f3eba658310b0..d1d6c314595b8 100644
--- a/src/frontend/planner_test/tests/testdata/output/create_source.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/create_source.yaml
@@ -6,7 +6,7 @@
 - id: create_source_without_connector
   sql: |
     create source s() with(a=1) FORMAT PLAIN ENCODE JSON;
-  planner_error: 'Invalid input syntax: schema definition is required for ENCODE JSON'
+  planner_error: 'Protocol error: missing field ''connector'''
 - id: create_source_without_schema_in_json
   sql: |
     create source s with(connector='kafka') FORMAT PLAIN ENCODE JSON;
diff --git a/src/frontend/planner_test/tests/testdata/output/grouping_sets.yaml b/src/frontend/planner_test/tests/testdata/output/grouping_sets.yaml
index 1ee513fd92746..ca4eb9f4f04ad 100644
--- a/src/frontend/planner_test/tests/testdata/output/grouping_sets.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/grouping_sets.yaml
@@ -179,3 +179,21 @@
         └─StreamExchange { dist: HashShard(items_sold.brand, items_sold.size, flag) }
           └─StreamExpand { column_subsets: [[], [items_sold.brand, items_sold.size], [items_sold.size], [items_sold.brand, items_sold.size]] }
             └─StreamTableScan { table: items_sold, columns: [items_sold.brand, items_sold.size, items_sold.sales, items_sold._row_id], pk: [items_sold._row_id], dist: UpstreamHashShard(items_sold._row_id) }
+- name: only one set in grouping sets
+  sql: |
+    create table items_sold (brand varchar, size varchar, sales int);
+    SELECT brand, sum(sales) FROM items_sold GROUP BY GROUPING SETS(brand);
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchProject { exprs: [items_sold.brand, sum(null:Int32)] }
+      └─BatchHashAgg { group_key: [items_sold.brand, 0:Int64], aggs: [sum(null:Int32)] }
+        └─BatchExchange { order: [], dist: HashShard(items_sold.brand, 0:Int64) }
+          └─BatchProject { exprs: [items_sold.brand, null:Int32, 0:Int64] }
+            └─BatchScan { table: items_sold, columns: [items_sold.brand], distribution: SomeShard }
+  stream_plan: |-
+    StreamMaterialize { columns: [brand, sum, 0:Int64(hidden)], stream_key: [brand, 0:Int64], pk_columns: [brand, 0:Int64], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [items_sold.brand, sum(null:Int32), 0:Int64] }
+      └─StreamHashAgg { group_key: [items_sold.brand, 0:Int64], aggs: [sum(null:Int32), count] }
+        └─StreamExchange { dist: HashShard(items_sold.brand, 0:Int64) }
+          └─StreamProject { exprs: [items_sold.brand, null:Int32, 0:Int64, items_sold._row_id] }
+            └─StreamTableScan { table: items_sold, columns: [items_sold.brand, items_sold._row_id], pk: [items_sold._row_id], dist: UpstreamHashShard(items_sold._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/project_set.yaml b/src/frontend/planner_test/tests/testdata/output/project_set.yaml
index 96b0d732e6ca0..23db668a070df 100644
--- a/src/frontend/planner_test/tests/testdata/output/project_set.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/project_set.yaml
@@ -191,3 +191,49 @@
     └─StreamProject { exprs: [1:Int32, t._row_id, projected_row_id] }
       └─StreamProjectSet { select_list: [1:Int32, Unnest($0), $1] }
         └─StreamTableScan { table: t, columns: [t.x, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) }
+- name: set-returning function disallowed in CASE WHEN
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select case when a1 is not null then unnest(a1) end from a;
+  binder_error: |-
+    Bind error: failed to bind expression: CASE WHEN a1 IS NOT NULL THEN unnest(a1) END
+
+    Caused by:
+      Bind error: table functions are not allowed in CASE
+- name: set-returning function disallowed in COALESCE
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select coalesce(unnest(a1), 0) from a;
+  binder_error: |-
+    Bind error: failed to bind expression: coalesce(unnest(a1), 0)
+
+    Caused by:
+      Bind error: table functions are not allowed in COALESCE
+- name: set-returning function disallowed in JOIN ON
+  sql: |
+    with a(a1) as (values (array[2]), (null)), b(b1) as (select 2) select a1 from a join b on unnest(a1) = b1;
+  binder_error: |-
+    Bind error: failed to bind expression: unnest(a1) = b1
+
+    Caused by:
+      Invalid input syntax: table functions are not allowed in JOIN ON
+- name: set-returning function disallowed in HAVING
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select count(*) from a group by a1 having unnest(a1) < 5;
+  binder_error: |-
+    Bind error: failed to bind expression: unnest(a1) < 5
+
+    Caused by:
+      Invalid input syntax: table functions are not allowed in HAVING
+- name: set-returning function disallowed in FILTER
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select count(*) filter(where unnest(a1) < 3) from a;
+  binder_error: |-
+    Bind error: failed to bind expression: count(*) FILTER(WHERE unnest(a1) < 3)
+
+    Caused by:
+      Invalid input syntax: table functions are not allowed in FILTER
+- name: set-returning function unsupported in GROUP BY yet
+  sql: |
+    with a(a1) as (values (array[2]), (null)) select count(*) from a group by unnest(a1);
+  planner_error: |-
+    Feature is not yet implemented: table function inside GROUP BY
+    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery.yaml b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
index 8131f64e3eea5..21c3cfb03847a 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
@@ -781,17 +781,16 @@
             │   └─BatchScan { table: integers, columns: [integers.i], distribution: SomeShard }
             └─BatchExchange { order: [], dist: HashShard(integers.i) }
               └─BatchProject { exprs: [integers.i, 1:Int32] }
-                └─BatchHashAgg { group_key: [integers.i, integers.i, flag], aggs: [] }
-                  └─BatchExchange { order: [], dist: HashShard(integers.i, integers.i, flag) }
-                    └─BatchExpand { column_subsets: [[integers.i, integers.i]] }
-                      └─BatchProject { exprs: [integers.i, integers.i] }
-                        └─BatchNestedLoopJoin { type: Inner, predicate: true, output: all }
-                          ├─BatchExchange { order: [], dist: Single }
-                          │ └─BatchHashAgg { group_key: [integers.i], aggs: [] }
-                          │   └─BatchExchange { order: [], dist: HashShard(integers.i) }
-                          │     └─BatchScan { table: integers, columns: [integers.i], distribution: SomeShard }
-                          └─BatchExchange { order: [], dist: Single }
-                            └─BatchScan { table: integers, columns: [], distribution: SomeShard }
+                └─BatchHashAgg { group_key: [integers.i, integers.i, 0:Int64], aggs: [] }
+                  └─BatchExchange { order: [], dist: HashShard(integers.i, integers.i, 0:Int64) }
+                    └─BatchProject { exprs: [integers.i, integers.i, 0:Int64] }
+                      └─BatchNestedLoopJoin { type: Inner, predicate: true, output: all }
+                        ├─BatchExchange { order: [], dist: Single }
+                        │ └─BatchHashAgg { group_key: [integers.i], aggs: [] }
+                        │   └─BatchExchange { order: [], dist: HashShard(integers.i) }
+                        │     └─BatchScan { table: integers, columns: [integers.i], distribution: SomeShard }
+                        └─BatchExchange { order: [], dist: Single }
+                          └─BatchScan { table: integers, columns: [], distribution: SomeShard }
 - name: test expand operator subquery 1
   sql: |
     CREATE TABLE integers(i INTEGER, correlated_col int);
diff --git a/src/frontend/planner_test/tests/testdata/output/watermark.yaml b/src/frontend/planner_test/tests/testdata/output/watermark.yaml
index a6f997735a00f..d1916a33192c6 100644
--- a/src/frontend/planner_test/tests/testdata/output/watermark.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/watermark.yaml
@@ -210,3 +210,12 @@
     └─StreamHopWindow { time_col: t.ts, slide: 00:01:00, size: 00:03:00, output: [window_start, t._row_id], output_watermarks: [window_start] }
       └─StreamFilter { predicate: IsNotNull(t.ts) }
         └─StreamTableScan { table: t, columns: [t.ts, t._row_id], pk: [t._row_id], dist: UpstreamHashShard(t._row_id) }
+- name: unnest
+  sql: |
+    create table t (ts timestamp with time zone, v1 int, watermark for ts as ts - INTERVAL '1' SECOND) append only;
+    explain create materialized view mv as select t.ts, unnest(Array[1,2,3]) from t emit on window close;
+  explain_output: |
+    StreamMaterialize { columns: [projected_row_id(hidden), ts, unnest, t._row_id(hidden)], stream_key: [t._row_id, projected_row_id], pk_columns: [t._row_id, projected_row_id], pk_conflict: NoCheck, watermark_columns: [ts] }
+    └─StreamEowcSort { sort_column: t.ts }
+      └─StreamProjectSet { select_list: [$0, Unnest(ARRAY[1, 2, 3]:List(Int32)), $1] }
+        └─StreamTableScan { table: t, columns: [ts, _row_id] }
diff --git a/src/frontend/src/binder/bind_context.rs b/src/frontend/src/binder/bind_context.rs
index e05f88e45d326..ca34eab3a3dbd 100644
--- a/src/frontend/src/binder/bind_context.rs
+++ b/src/frontend/src/binder/bind_context.rs
@@ -19,6 +19,7 @@ use std::rc::Rc;
 use parse_display::Display;
 use risingwave_common::catalog::Field;
 use risingwave_common::error::{ErrorCode, Result};
+use risingwave_common::types::DataType;
 use risingwave_sqlparser::ast::TableAlias;
 
 type LiteResult<T> = std::result::Result<T, ErrorCode>;
@@ -79,6 +80,8 @@ pub struct BindContext {
     /// Map the cte's name to its Relation::Subquery.
     /// The `ShareId` of the value is used to help the planner identify the share plan.
     pub cte_to_relation: HashMap<String, Rc<(ShareId, BoundQuery, TableAlias)>>,
+    /// Current lambda functions's arguments
+    pub lambda_args: Option<HashMap<String, (usize, DataType)>>,
 }
 
 /// Holds the context for the `BindContext`'s `ColumnGroup`s.
diff --git a/src/frontend/src/binder/bind_param.rs b/src/frontend/src/binder/bind_param.rs
index 621d78c845767..44c11f62393ae 100644
--- a/src/frontend/src/binder/bind_param.rs
+++ b/src/frontend/src/binder/bind_param.rs
@@ -49,6 +49,9 @@ impl ExprRewriter for ParamRewriter {
             ExprImpl::InputRef(inner) => self.rewrite_input_ref(*inner),
             ExprImpl::Literal(inner) => self.rewrite_literal(*inner),
             ExprImpl::FunctionCall(inner) => self.rewrite_function_call(*inner),
+            ExprImpl::FunctionCallWithLambda(inner) => {
+                self.rewrite_function_call_with_lambda(*inner)
+            }
             ExprImpl::AggCall(inner) => self.rewrite_agg_call(*inner),
             ExprImpl::Subquery(inner) => self.rewrite_subquery(*inner),
             ExprImpl::CorrelatedInputRef(inner) => self.rewrite_correlated_input_ref(*inner),
diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs
index e778e3aaeaff1..4cb703906bf8c 100644
--- a/src/frontend/src/binder/expr/function.rs
+++ b/src/frontend/src/binder/expr/function.rs
@@ -31,15 +31,15 @@ use risingwave_expr::function::window::{
     Frame, FrameBound, FrameBounds, FrameExclusion, WindowFuncKind,
 };
 use risingwave_sqlparser::ast::{
-    Function, FunctionArg, FunctionArgExpr, WindowFrameBound, WindowFrameExclusion,
+    self, Function, FunctionArg, FunctionArgExpr, Ident, WindowFrameBound, WindowFrameExclusion,
     WindowFrameUnits, WindowSpec,
 };
 
 use crate::binder::bind_context::Clause;
 use crate::binder::{Binder, BoundQuery, BoundSetExpr};
 use crate::expr::{
-    AggCall, Expr, ExprImpl, ExprType, FunctionCall, Literal, Now, OrderBy, Subquery, SubqueryKind,
-    TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction,
+    AggCall, Expr, ExprImpl, ExprType, FunctionCall, FunctionCallWithLambda, Literal, Now, OrderBy,
+    Subquery, SubqueryKind, TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction,
 };
 use crate::utils::Condition;
 
@@ -101,6 +101,11 @@ impl Binder {
             return Ok(ExprImpl::literal_varchar("".to_string()));
         }
 
+        if function_name == "array_transform" {
+            // For type inference, we need to bind the array type first.
+            return self.bind_array_transform(f);
+        }
+
         let inputs = f
             .args
             .into_iter()
@@ -154,6 +159,75 @@ impl Binder {
         self.bind_builtin_scalar_function(function_name.as_str(), inputs)
     }
 
+    fn bind_array_transform(&mut self, f: Function) -> Result<ExprImpl> {
+        let [array, lambda] = <[FunctionArg; 2]>::try_from(f.args).map_err(|args| -> RwError {
+            ErrorCode::BindError(format!(
+                "`array_transform` expect two inputs `array` and `lambda`, but {} were given",
+                args.len()
+            ))
+            .into()
+        })?;
+
+        let bound_array = self.bind_function_arg(array)?;
+        let [bound_array] = <[ExprImpl; 1]>::try_from(bound_array).map_err(|bound_array| -> RwError {
+            ErrorCode::BindError(format!("The `array` argument for `array_transform` should be bound to one argument, but {} were got", bound_array.len()))
+                .into()
+        })?;
+
+        let inner_ty = match bound_array.return_type() {
+            DataType::List(ty) => *ty,
+            real_type => {
+                return Err(ErrorCode::BindError(format!(
+                "The `array` argument for `array_transform` should be an array, but {} were got",
+                real_type
+            ))
+                .into())
+            }
+        };
+
+        let ast::FunctionArgExpr::Expr(ast::Expr::LambdaFunction { args: lambda_args, body: lambda_body }) = lambda.get_expr() else {
+            return Err(ErrorCode::BindError(
+                "The `lambda` argument for `array_transform` should be a lambda function".to_string()
+            ).into());
+        };
+
+        let [lambda_arg] = <[Ident; 1]>::try_from(lambda_args).map_err(|args| -> RwError {
+            ErrorCode::BindError(format!(
+                "The `lambda` argument for `array_transform` should be a lambda function with one argument, but {} were given",
+                args.len()
+            ))
+            .into()
+        })?;
+
+        let bound_lambda = self.bind_unary_lambda_function(inner_ty, lambda_arg, *lambda_body)?;
+
+        let lambda_ret_type = bound_lambda.return_type();
+        let transform_ret_type = DataType::List(Box::new(lambda_ret_type));
+
+        Ok(ExprImpl::FunctionCallWithLambda(Box::new(
+            FunctionCallWithLambda::new_unchecked(
+                ExprType::ArrayTransform,
+                vec![bound_array],
+                bound_lambda,
+                transform_ret_type,
+            ),
+        )))
+    }
+
+    fn bind_unary_lambda_function(
+        &mut self,
+        input_ty: DataType,
+        arg: Ident,
+        body: ast::Expr,
+    ) -> Result<ExprImpl> {
+        let lambda_args = HashMap::from([(arg.real_value(), (0usize, input_ty))]);
+        let orig_lambda_args = self.context.lambda_args.replace(lambda_args);
+        let body = self.bind_expr_inner(body)?;
+        self.context.lambda_args = orig_lambda_args;
+
+        Ok(body)
+    }
+
     pub(super) fn bind_agg(&mut self, f: Function, kind: AggKind) -> Result<ExprImpl> {
         self.ensure_aggregate_allowed()?;
 
@@ -588,7 +662,12 @@ impl Binder {
                     "boolne",
                     rewrite(ExprType::NotEqual, Binder::rewrite_two_bool_inputs),
                 ),
-                ("coalesce", raw_call(ExprType::Coalesce)),
+                ("coalesce", rewrite(ExprType::Coalesce, |inputs| {
+                    if inputs.iter().any(ExprImpl::has_table_function) {
+                        return Err(ErrorCode::BindError("table functions are not allowed in COALESCE".into()).into());
+                    }
+                    Ok(inputs)
+                })),
                 (
                     "nullif",
                     rewrite(ExprType::Case, Binder::rewrite_nullif_to_case_when),
@@ -680,6 +759,8 @@ impl Binder {
                 ("octet_length", raw_call(ExprType::OctetLength)),
                 ("bit_length", raw_call(ExprType::BitLength)),
                 ("regexp_match", raw_call(ExprType::RegexpMatch)),
+                ("regexp_replace", raw_call(ExprType::RegexpReplace)),
+                ("regexp_count", raw_call(ExprType::RegexpCount)),
                 ("chr", raw_call(ExprType::Chr)),
                 ("starts_with", raw_call(ExprType::StartsWith)),
                 ("initcap", raw_call(ExprType::Initcap)),
@@ -707,10 +788,12 @@ impl Binder {
                 ("array_prepend", raw_call(ExprType::ArrayPrepend)),
                 ("array_to_string", raw_call(ExprType::ArrayToString)),
                 ("array_distinct", raw_call(ExprType::ArrayDistinct)),
+                ("array_min", raw_call(ExprType::ArrayMin)),
                 ("array_length", raw_call(ExprType::ArrayLength)),
                 ("cardinality", raw_call(ExprType::Cardinality)),
                 ("array_remove", raw_call(ExprType::ArrayRemove)),
                 ("array_replace", raw_call(ExprType::ArrayReplace)),
+                ("array_max", raw_call(ExprType::ArrayMax)),
                 ("array_position", raw_call(ExprType::ArrayPosition)),
                 ("array_positions", raw_call(ExprType::ArrayPositions)),
                 ("trim_array", raw_call(ExprType::TrimArray)),
@@ -1288,18 +1371,19 @@ impl Binder {
     fn ensure_table_function_allowed(&self) -> Result<()> {
         if let Some(clause) = self.context.clause {
             match clause {
-                Clause::Where | Clause::Values | Clause::GeneratedColumn => {
+                Clause::JoinOn
+                | Clause::Where
+                | Clause::Having
+                | Clause::Filter
+                | Clause::Values
+                | Clause::GeneratedColumn => {
                     return Err(ErrorCode::InvalidInputSyntax(format!(
                         "table functions are not allowed in {}",
                         clause
                     ))
                     .into());
                 }
-                Clause::JoinOn
-                | Clause::GroupBy
-                | Clause::Having
-                | Clause::Filter
-                | Clause::From => {}
+                Clause::GroupBy | Clause::From => {}
             }
         }
         Ok(())
diff --git a/src/frontend/src/binder/expr/mod.rs b/src/frontend/src/binder/expr/mod.rs
index ce9e7ba4fa374..7b80dcc3c5495 100644
--- a/src/frontend/src/binder/expr/mod.rs
+++ b/src/frontend/src/binder/expr/mod.rs
@@ -24,7 +24,7 @@ use risingwave_sqlparser::ast::{
 
 use crate::binder::expr::function::SYS_FUNCTION_WITHOUT_ARGS;
 use crate::binder::Binder;
-use crate::expr::{Expr as _, ExprImpl, ExprType, FunctionCall, Parameter, SubqueryKind};
+use crate::expr::{Expr as _, ExprImpl, ExprType, FunctionCall, InputRef, Parameter, SubqueryKind};
 
 mod binary_op;
 mod column;
@@ -97,6 +97,17 @@ impl Binder {
                     // NOTE: Here we don't 100% follow the behavior of Postgres, as it doesn't
                     // allow `session_user()` while we do.
                     self.bind_function(Function::no_arg(ObjectName(vec![ident])))
+                } else if let Some(ref lambda_args) = self.context.lambda_args {
+                    // We don't support capture, so if the expression is in the lambda context,
+                    // we'll not bind it for table columns.
+                    if let Some((arg_idx, arg_type)) = lambda_args.get(&ident.real_value()) {
+                        Ok(InputRef::new(*arg_idx, arg_type.clone()).into())
+                    } else {
+                        Err(
+                            ErrorCode::ItemNotFound(format!("Unknown arg: {}", ident.real_value()))
+                                .into(),
+                        )
+                    }
                 } else {
                     self.bind_column(&[ident])
                 }
@@ -469,6 +480,11 @@ impl Binder {
         if let Some(expr) = else_result_expr {
             inputs.push(expr);
         }
+        if inputs.iter().any(ExprImpl::has_table_function) {
+            return Err(
+                ErrorCode::BindError("table functions are not allowed in CASE".into()).into(),
+            );
+        }
         Ok(FunctionCall::new(ExprType::Case, inputs)?.into())
     }
 
diff --git a/src/frontend/src/binder/relation/table_function.rs b/src/frontend/src/binder/relation/table_function.rs
index 1be11687fb1c8..1b53364e2dab5 100644
--- a/src/frontend/src/binder/relation/table_function.rs
+++ b/src/frontend/src/binder/relation/table_function.rs
@@ -32,8 +32,8 @@ use crate::expr::Expr;
 impl Binder {
     /// Binds a table function AST, which is a function call in a relation position.
     ///
-    /// Besides [`TableFunction`] expr, it can also be other things like window table functions, or
-    /// scalar functions.
+    /// Besides [`crate::expr::TableFunction`] expr, it can also be other things like window table
+    /// functions, or scalar functions.
     pub(super) fn bind_table_function(
         &mut self,
         name: ObjectName,
diff --git a/src/frontend/src/catalog/system_catalog/information_schema/mod.rs b/src/frontend/src/catalog/system_catalog/information_schema/mod.rs
index 4d6641347aba9..ec0a497851a14 100644
--- a/src/frontend/src/catalog/system_catalog/information_schema/mod.rs
+++ b/src/frontend/src/catalog/system_catalog/information_schema/mod.rs
@@ -14,6 +14,8 @@
 
 pub mod columns;
 pub mod tables;
+pub mod views;
 
 pub use columns::*;
 pub use tables::*;
+pub use views::*;
diff --git a/src/frontend/src/catalog/system_catalog/information_schema/views.rs b/src/frontend/src/catalog/system_catalog/information_schema/views.rs
new file mode 100644
index 0000000000000..f13ec5a9585b0
--- /dev/null
+++ b/src/frontend/src/catalog/system_catalog/information_schema/views.rs
@@ -0,0 +1,45 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::LazyLock;
+
+use risingwave_common::catalog::INFORMATION_SCHEMA_SCHEMA_NAME;
+use risingwave_common::types::DataType;
+
+use crate::catalog::system_catalog::BuiltinView;
+
+/// The view `views` contains all views defined in the current database. Only those views
+/// are shown that the current user has access to (by way of being the owner or having
+/// some privilege).
+/// Ref: [`https://www.postgresql.org/docs/current/infoschema-views.html`]
+///
+/// In RisingWave, `views` contains information about defined views.
+pub static INFORMATION_SCHEMA_VIEWS: LazyLock<BuiltinView> = LazyLock::new(|| BuiltinView {
+    name: "views",
+    schema: INFORMATION_SCHEMA_SCHEMA_NAME,
+    columns: &[
+        (DataType::Varchar, "table_catalog"),
+        (DataType::Varchar, "table_schema"),
+        (DataType::Varchar, "table_name"),
+        (DataType::Varchar, "view_definition"),
+    ],
+    sql: "SELECT CURRENT_DATABASE() AS table_catalog, \
+                s.name AS table_schema, \
+                v.name AS table_name, \
+                v.definition AS view_definition \
+            FROM rw_catalog.rw_views v \
+            JOIN rw_catalog.rw_schemas s ON v.schema_id = s.id \
+        ORDER BY table_schema, table_name"
+        .to_string(),
+});
diff --git a/src/frontend/src/catalog/system_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/mod.rs
index 892c0f9c35570..5d4a785bd4513 100644
--- a/src/frontend/src/catalog/system_catalog/mod.rs
+++ b/src/frontend/src/catalog/system_catalog/mod.rs
@@ -377,6 +377,7 @@ prepare_sys_catalog! {
     { BuiltinCatalog::View(&PG_DEPEND) },
     { BuiltinCatalog::View(&INFORMATION_SCHEMA_COLUMNS) },
     { BuiltinCatalog::View(&INFORMATION_SCHEMA_TABLES) },
+    { BuiltinCatalog::View(&INFORMATION_SCHEMA_VIEWS) },
     { BuiltinCatalog::Table(&RW_DATABASES), read_rw_database_info },
     { BuiltinCatalog::Table(&RW_SCHEMAS), read_rw_schema_info },
     { BuiltinCatalog::Table(&RW_USERS), read_rw_user_info },
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_ddl_progress.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_ddl_progress.rs
index a7f32008dd39b..a147f96bb8c11 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_ddl_progress.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_ddl_progress.rs
@@ -17,6 +17,7 @@ use risingwave_common::catalog::RW_CATALOG_SCHEMA_NAME;
 use risingwave_common::error::Result;
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::{DataType, ScalarImpl};
+use risingwave_common::util::epoch::Epoch;
 
 use crate::catalog::system_catalog::{BuiltinTable, SysCatalogReaderImpl};
 
@@ -27,22 +28,34 @@ pub const RW_DDL_PROGRESS: BuiltinTable = BuiltinTable {
         (DataType::Int64, "ddl_id"),
         (DataType::Varchar, "ddl_statement"),
         (DataType::Varchar, "progress"),
+        (DataType::Timestamptz, "initialized_at"),
     ],
     pk: &[0],
 };
 
 impl SysCatalogReaderImpl {
     pub async fn read_ddl_progress(&self) -> Result<Vec<OwnedRow>> {
-        let ddl_progress = self
-            .meta_client
-            .list_ddl_progress()
-            .await?
+        let ddl_progresses = self.meta_client.list_ddl_progress().await?;
+
+        let table_ids = ddl_progresses
+            .iter()
+            .map(|progress| progress.id as u32)
+            .collect_vec();
+
+        let tables = self.meta_client.get_tables(&table_ids).await?;
+
+        let ddl_progress = ddl_progresses
             .into_iter()
             .map(|s| {
+                let initialized_at = tables
+                    .get(&(s.id as u32))
+                    .and_then(|table| table.initialized_at_epoch.map(Epoch::from));
+
                 OwnedRow::new(vec![
                     Some(ScalarImpl::Int64(s.id as i64)),
                     Some(ScalarImpl::Utf8(s.statement.into())),
                     Some(ScalarImpl::Utf8(s.progress.into())),
+                    initialized_at.map(|e| e.as_scalar()),
                 ])
             })
             .collect_vec();
diff --git a/src/frontend/src/catalog/table_catalog.rs b/src/frontend/src/catalog/table_catalog.rs
index 1bea5d5a07d5c..6c83df13e80be 100644
--- a/src/frontend/src/catalog/table_catalog.rs
+++ b/src/frontend/src/catalog/table_catalog.rs
@@ -140,7 +140,7 @@ pub struct TableCatalog {
     pub watermark_columns: FixedBitSet,
 
     /// Optional field specifies the distribution key indices in pk.
-    /// See https://github.com/risingwavelabs/risingwave/issues/8377 for more information.
+    /// See <https://github.com/risingwavelabs/risingwave/issues/8377> for more information.
     pub dist_key_in_pk: Vec<usize>,
 
     pub created_at_epoch: Option<Epoch>,
diff --git a/src/frontend/src/expr/expr_mutator.rs b/src/frontend/src/expr/expr_mutator.rs
index d92e854c24849..5ec724ff79866 100644
--- a/src/frontend/src/expr/expr_mutator.rs
+++ b/src/frontend/src/expr/expr_mutator.rs
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use super::{
-    AggCall, CorrelatedInputRef, ExprImpl, FunctionCall, InputRef, Literal, Now, Parameter,
-    Subquery, TableFunction, UserDefinedFunction, WindowFunction,
+    AggCall, CorrelatedInputRef, ExprImpl, FunctionCall, FunctionCallWithLambda, InputRef, Literal,
+    Now, Parameter, Subquery, TableFunction, UserDefinedFunction, WindowFunction,
 };
 
 /// with the same visit logic of `ExprVisitor`, but mutable.
@@ -24,6 +24,7 @@ pub trait ExprMutator {
             ExprImpl::InputRef(inner) => self.visit_input_ref(inner),
             ExprImpl::Literal(inner) => self.visit_literal(inner),
             ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
+            ExprImpl::FunctionCallWithLambda(inner) => self.visit_function_call_with_lambda(inner),
             ExprImpl::AggCall(inner) => self.visit_agg_call(inner),
             ExprImpl::Subquery(inner) => self.visit_subquery(inner),
             ExprImpl::CorrelatedInputRef(inner) => self.visit_correlated_input_ref(inner),
@@ -40,6 +41,11 @@ pub trait ExprMutator {
             .iter_mut()
             .for_each(|expr| self.visit_expr(expr))
     }
+
+    fn visit_function_call_with_lambda(&mut self, func_call: &mut FunctionCallWithLambda) {
+        self.visit_function_call(func_call.base_mut())
+    }
+
     fn visit_agg_call(&mut self, agg_call: &mut AggCall) {
         agg_call
             .args_mut()
diff --git a/src/frontend/src/expr/expr_rewriter.rs b/src/frontend/src/expr/expr_rewriter.rs
index b75e5ce7ba791..b93a67cd34101 100644
--- a/src/frontend/src/expr/expr_rewriter.rs
+++ b/src/frontend/src/expr/expr_rewriter.rs
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use super::{
-    AggCall, CorrelatedInputRef, ExprImpl, FunctionCall, InputRef, Literal, Parameter, Subquery,
-    TableFunction, UserDefinedFunction, WindowFunction,
+    AggCall, CorrelatedInputRef, ExprImpl, FunctionCall, FunctionCallWithLambda, InputRef, Literal,
+    Parameter, Subquery, TableFunction, UserDefinedFunction, WindowFunction,
 };
 use crate::expr::Now;
 
@@ -27,6 +27,9 @@ pub trait ExprRewriter {
             ExprImpl::InputRef(inner) => self.rewrite_input_ref(*inner),
             ExprImpl::Literal(inner) => self.rewrite_literal(*inner),
             ExprImpl::FunctionCall(inner) => self.rewrite_function_call(*inner),
+            ExprImpl::FunctionCallWithLambda(inner) => {
+                self.rewrite_function_call_with_lambda(*inner)
+            }
             ExprImpl::AggCall(inner) => self.rewrite_agg_call(*inner),
             ExprImpl::Subquery(inner) => self.rewrite_subquery(*inner),
             ExprImpl::CorrelatedInputRef(inner) => self.rewrite_correlated_input_ref(*inner),
@@ -37,6 +40,7 @@ pub trait ExprRewriter {
             ExprImpl::Now(inner) => self.rewrite_now(*inner),
         }
     }
+
     fn rewrite_function_call(&mut self, func_call: FunctionCall) -> ExprImpl {
         let (func_type, inputs, ret) = func_call.decompose();
         let inputs = inputs
@@ -45,6 +49,16 @@ pub trait ExprRewriter {
             .collect();
         FunctionCall::new_unchecked(func_type, inputs, ret).into()
     }
+
+    fn rewrite_function_call_with_lambda(&mut self, func_call: FunctionCallWithLambda) -> ExprImpl {
+        let (func_type, inputs, lambda_arg, ret) = func_call.into_parts();
+        let inputs = inputs
+            .into_iter()
+            .map(|expr| self.rewrite_expr(expr))
+            .collect();
+        FunctionCallWithLambda::new_unchecked(func_type, inputs, lambda_arg, ret).into()
+    }
+
     fn rewrite_agg_call(&mut self, agg_call: AggCall) -> ExprImpl {
         let (func_type, inputs, distinct, order_by, filter, direct_args) = agg_call.decompose();
         let inputs = inputs
@@ -57,21 +71,27 @@ pub trait ExprRewriter {
             .unwrap()
             .into()
     }
+
     fn rewrite_parameter(&mut self, parameter: Parameter) -> ExprImpl {
         parameter.into()
     }
+
     fn rewrite_literal(&mut self, literal: Literal) -> ExprImpl {
         literal.into()
     }
+
     fn rewrite_input_ref(&mut self, input_ref: InputRef) -> ExprImpl {
         input_ref.into()
     }
+
     fn rewrite_subquery(&mut self, subquery: Subquery) -> ExprImpl {
         subquery.into()
     }
+
     fn rewrite_correlated_input_ref(&mut self, input_ref: CorrelatedInputRef) -> ExprImpl {
         input_ref.into()
     }
+
     fn rewrite_table_function(&mut self, table_func: TableFunction) -> ExprImpl {
         let TableFunction {
             args,
@@ -91,6 +111,7 @@ pub trait ExprRewriter {
         }
         .into()
     }
+
     fn rewrite_window_function(&mut self, window_func: WindowFunction) -> ExprImpl {
         let WindowFunction {
             args,
@@ -114,6 +135,7 @@ pub trait ExprRewriter {
         }
         .into()
     }
+
     fn rewrite_user_defined_function(&mut self, udf: UserDefinedFunction) -> ExprImpl {
         let UserDefinedFunction { args, catalog } = udf;
         let args = args
@@ -122,6 +144,7 @@ pub trait ExprRewriter {
             .collect();
         UserDefinedFunction { args, catalog }.into()
     }
+
     fn rewrite_now(&mut self, now: Now) -> ExprImpl {
         now.into()
     }
diff --git a/src/frontend/src/expr/expr_visitor.rs b/src/frontend/src/expr/expr_visitor.rs
index 3e12399d7f5cf..4fdfbb07f7518 100644
--- a/src/frontend/src/expr/expr_visitor.rs
+++ b/src/frontend/src/expr/expr_visitor.rs
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use super::{
-    AggCall, CorrelatedInputRef, ExprImpl, FunctionCall, InputRef, Literal, Now, Parameter,
-    Subquery, TableFunction, UserDefinedFunction, WindowFunction,
+    AggCall, CorrelatedInputRef, ExprImpl, FunctionCall, FunctionCallWithLambda, InputRef, Literal,
+    Now, Parameter, Subquery, TableFunction, UserDefinedFunction, WindowFunction,
 };
 
 /// Traverse an expression tree.
@@ -36,6 +36,7 @@ pub trait ExprVisitor<R: Default> {
             ExprImpl::InputRef(inner) => self.visit_input_ref(inner),
             ExprImpl::Literal(inner) => self.visit_literal(inner),
             ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
+            ExprImpl::FunctionCallWithLambda(inner) => self.visit_function_call_with_lambda(inner),
             ExprImpl::AggCall(inner) => self.visit_agg_call(inner),
             ExprImpl::Subquery(inner) => self.visit_subquery(inner),
             ExprImpl::CorrelatedInputRef(inner) => self.visit_correlated_input_ref(inner),
@@ -54,6 +55,9 @@ pub trait ExprVisitor<R: Default> {
             .reduce(Self::merge)
             .unwrap_or_default()
     }
+    fn visit_function_call_with_lambda(&mut self, func_call: &FunctionCallWithLambda) -> R {
+        self.visit_function_call(func_call.base())
+    }
     fn visit_agg_call(&mut self, agg_call: &AggCall) -> R {
         let mut r = agg_call
             .args()
diff --git a/src/frontend/src/expr/function_call.rs b/src/frontend/src/expr/function_call.rs
index d71815e155821..e826d43d9342a 100644
--- a/src/frontend/src/expr/function_call.rs
+++ b/src/frontend/src/expr/function_call.rs
@@ -25,9 +25,9 @@ use crate::expr::{ExprDisplay, ExprType, ExprVisitor, ImpureAnalyzer};
 
 #[derive(Clone, Eq, PartialEq, Hash)]
 pub struct FunctionCall {
-    func_type: ExprType,
-    return_type: DataType,
-    inputs: Vec<ExprImpl>,
+    pub(super) func_type: ExprType,
+    pub(super) return_type: DataType,
+    pub(super) inputs: Vec<ExprImpl>,
 }
 
 fn debug_binary_op(
@@ -102,11 +102,7 @@ impl FunctionCall {
     // [elsewhere](crate::expr::type_inference::build_type_derive_map).
     pub fn new(func_type: ExprType, mut inputs: Vec<ExprImpl>) -> RwResult<Self> {
         let return_type = infer_type(func_type, &mut inputs)?;
-        Ok(Self {
-            func_type,
-            return_type,
-            inputs,
-        })
+        Ok(Self::new_unchecked(func_type, inputs, return_type))
     }
 
     /// Create a cast expr over `child` to `target` type in `allows` context.
@@ -153,12 +149,7 @@ impl FunctionCall {
         } else if child.is_untyped() || cast_ok(&source, &target, allows) {
             // Always Ok below. Safe to mutate `child`.
             let owned = std::mem::replace(child, ExprImpl::literal_bool(false));
-            *child = Self {
-                func_type: ExprType::Cast,
-                return_type: target,
-                inputs: vec![owned],
-            }
-            .into();
+            *child = Self::new_unchecked(ExprType::Cast, vec![owned], target).into();
             Ok(())
         } else {
             Err(CastError(format!(
@@ -170,7 +161,7 @@ impl FunctionCall {
 
     /// Cast a `ROW` expression to the target type. We intentionally disallow casting arbitrary
     /// expressions, like `ROW(1)::STRUCT<i INTEGER>` to `STRUCT<VARCHAR>`, although an integer
-    /// is castible to VARCHAR. It's to simply the casting rules.
+    /// is castable to VARCHAR. It's to simply the casting rules.
     fn cast_row_expr(
         func: &mut FunctionCall,
         target_type: DataType,
diff --git a/src/frontend/src/expr/function_call_with_lambda.rs b/src/frontend/src/expr/function_call_with_lambda.rs
new file mode 100644
index 0000000000000..97a97864198d6
--- /dev/null
+++ b/src/frontend/src/expr/function_call_with_lambda.rs
@@ -0,0 +1,117 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::types::DataType;
+
+use super::{ExprImpl, FunctionCall};
+use crate::expr::{Expr, ExprType};
+
+/// Similar to [`FunctionCall`], with an extra lambda function argument.
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct FunctionCallWithLambda {
+    base: FunctionCall,
+    lambda_arg: ExprImpl,
+}
+
+impl std::fmt::Debug for FunctionCallWithLambda {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if f.alternate() {
+            f.debug_struct("FunctionCallWithLambda")
+                .field("func_type", &self.base.func_type)
+                .field("return_type", &self.base.return_type)
+                .field("inputs", &self.base.inputs)
+                .field("lambda_arg", &self.lambda_arg)
+                .finish()
+        } else {
+            let func_name = format!("{:?}", self.base.func_type);
+            let mut builder = f.debug_tuple(&func_name);
+            for input in &self.base.inputs {
+                builder.field(input);
+            }
+            builder.field(&self.lambda_arg);
+            builder.finish()
+        }
+    }
+}
+
+impl FunctionCallWithLambda {
+    pub fn new_unchecked(
+        func_type: ExprType,
+        inputs: Vec<ExprImpl>,
+        lambda_arg: ExprImpl,
+        return_type: DataType,
+    ) -> Self {
+        assert!([ExprType::ArrayTransform].contains(&func_type));
+        Self {
+            base: FunctionCall::new_unchecked(func_type, inputs, return_type),
+            lambda_arg,
+        }
+    }
+
+    pub fn inputs(&self) -> &[ExprImpl] {
+        self.base.inputs()
+    }
+
+    pub fn func_type(&self) -> ExprType {
+        self.base.func_type()
+    }
+
+    pub fn return_type(&self) -> DataType {
+        self.base.return_type()
+    }
+
+    pub fn base(&self) -> &FunctionCall {
+        &self.base
+    }
+
+    pub fn base_mut(&mut self) -> &mut FunctionCall {
+        &mut self.base
+    }
+
+    pub fn inputs_with_lambda_arg(&self) -> impl Iterator<Item = &'_ ExprImpl> {
+        self.inputs().iter().chain([&self.lambda_arg])
+    }
+
+    pub fn to_full_function_call(&self) -> FunctionCall {
+        let full_inputs = self.inputs_with_lambda_arg().cloned();
+        FunctionCall::new_unchecked(self.func_type(), full_inputs.collect(), self.return_type())
+    }
+
+    pub fn into_parts(self) -> (ExprType, Vec<ExprImpl>, ExprImpl, DataType) {
+        let Self { base, lambda_arg } = self;
+        let (func_type, inputs, return_type) = base.decompose();
+        (func_type, inputs, lambda_arg, return_type)
+    }
+}
+
+impl Expr for FunctionCallWithLambda {
+    fn return_type(&self) -> DataType {
+        self.base.return_type()
+    }
+
+    fn to_expr_proto(&self) -> risingwave_pb::expr::ExprNode {
+        use risingwave_pb::expr::expr_node::*;
+        use risingwave_pb::expr::*;
+        ExprNode {
+            function_type: self.func_type().into(),
+            return_type: Some(self.return_type().to_protobuf()),
+            rex_node: Some(RexNode::FuncCall(FunctionCall {
+                children: self
+                    .inputs_with_lambda_arg()
+                    .map(Expr::to_expr_proto)
+                    .collect(),
+            })),
+        }
+    }
+}
diff --git a/src/frontend/src/expr/mod.rs b/src/frontend/src/expr/mod.rs
index 6e1e206ccd6ab..d999fcbe4c1e8 100644
--- a/src/frontend/src/expr/mod.rs
+++ b/src/frontend/src/expr/mod.rs
@@ -29,6 +29,7 @@ use risingwave_pb::expr::{ExprNode, ProjectSetSelectItem};
 mod agg_call;
 mod correlated_input_ref;
 mod function_call;
+mod function_call_with_lambda;
 mod input_ref;
 mod literal;
 mod now;
@@ -55,6 +56,7 @@ pub use expr_mutator::ExprMutator;
 pub use expr_rewriter::ExprRewriter;
 pub use expr_visitor::ExprVisitor;
 pub use function_call::{is_row_function, FunctionCall, FunctionCallDisplay};
+pub use function_call_with_lambda::FunctionCallWithLambda;
 pub use input_ref::{input_ref_to_column_indices, InputRef, InputRefDisplay};
 pub use literal::Literal;
 pub use now::{InlineNowProcTime, Now};
@@ -117,6 +119,7 @@ impl_expr_impl!(
     InputRef,
     Literal,
     FunctionCall,
+    FunctionCallWithLambda,
     AggCall,
     Subquery,
     TableFunction,
@@ -364,7 +367,7 @@ macro_rules! impl_has_variant {
     };
 }
 
-impl_has_variant! {InputRef, Literal, FunctionCall, AggCall, Subquery, TableFunction, WindowFunction}
+impl_has_variant! {InputRef, Literal, FunctionCall, FunctionCallWithLambda, AggCall, Subquery, TableFunction, WindowFunction}
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct InequalityInputPair {
@@ -598,8 +601,6 @@ impl ExprImpl {
 
                 fn visit_expr(&mut self, expr: &ExprImpl) {
                     match expr {
-                        ExprImpl::Literal(_inner) => {}
-                        ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
                         ExprImpl::CorrelatedInputRef(_)
                         | ExprImpl::InputRef(_)
                         | ExprImpl::AggCall(_)
@@ -609,6 +610,11 @@ impl ExprImpl {
                         | ExprImpl::UserDefinedFunction(_)
                         | ExprImpl::Parameter(_)
                         | ExprImpl::Now(_) => self.has_others = true,
+                        ExprImpl::Literal(_inner) => {}
+                        ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
+                        ExprImpl::FunctionCallWithLambda(inner) => {
+                            self.visit_function_call_with_lambda(inner)
+                        }
                     }
                 }
             }
@@ -989,6 +995,9 @@ impl std::fmt::Debug for ExprImpl {
                 Self::InputRef(arg0) => f.debug_tuple("InputRef").field(arg0).finish(),
                 Self::Literal(arg0) => f.debug_tuple("Literal").field(arg0).finish(),
                 Self::FunctionCall(arg0) => f.debug_tuple("FunctionCall").field(arg0).finish(),
+                Self::FunctionCallWithLambda(arg0) => {
+                    f.debug_tuple("FunctionCallWithLambda").field(arg0).finish()
+                }
                 Self::AggCall(arg0) => f.debug_tuple("AggCall").field(arg0).finish(),
                 Self::Subquery(arg0) => f.debug_tuple("Subquery").field(arg0).finish(),
                 Self::CorrelatedInputRef(arg0) => {
@@ -1007,6 +1016,7 @@ impl std::fmt::Debug for ExprImpl {
             Self::InputRef(x) => write!(f, "{:?}", x),
             Self::Literal(x) => write!(f, "{:?}", x),
             Self::FunctionCall(x) => write!(f, "{:?}", x),
+            Self::FunctionCallWithLambda(x) => write!(f, "{:?}", x),
             Self::AggCall(x) => write!(f, "{:?}", x),
             Self::Subquery(x) => write!(f, "{:?}", x),
             Self::CorrelatedInputRef(x) => write!(f, "{:?}", x),
@@ -1045,6 +1055,14 @@ impl std::fmt::Debug for ExprDisplay<'_> {
                     input_schema: self.input_schema
                 }
             ),
+            ExprImpl::FunctionCallWithLambda(x) => write!(
+                f,
+                "{:?}",
+                FunctionCallDisplay {
+                    function_call: &x.to_full_function_call(),
+                    input_schema: self.input_schema
+                }
+            ),
             ExprImpl::AggCall(x) => write!(f, "{:?}", x),
             ExprImpl::Subquery(x) => write!(f, "{:?}", x),
             ExprImpl::CorrelatedInputRef(x) => write!(f, "{:?}", x),
diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs
index c858a7a2e6049..bb2e1d12d8f87 100644
--- a/src/frontend/src/expr/pure.rs
+++ b/src/frontend/src/expr/pure.rs
@@ -103,6 +103,8 @@ impl ExprVisitor<bool> for ImpureAnalyzer {
             | expr_node::Type::BitLength
             | expr_node::Type::Overlay
             | expr_node::Type::RegexpMatch
+            | expr_node::Type::RegexpReplace
+            | expr_node::Type::RegexpCount
             | expr_node::Type::Pow
             | expr_node::Type::Exp
             | expr_node::Type::Ln
@@ -151,10 +153,12 @@ impl ExprVisitor<bool> for ImpureAnalyzer {
             | expr_node::Type::Row
             | expr_node::Type::ArrayToString
             | expr_node::Type::ArrayCat
+            | expr_node::Type::ArrayMax
             | expr_node::Type::ArrayAppend
             | expr_node::Type::ArrayPrepend
             | expr_node::Type::FormatType
             | expr_node::Type::ArrayDistinct
+            | expr_node::Type::ArrayMin
             | expr_node::Type::ArrayDims
             | expr_node::Type::ArrayLength
             | expr_node::Type::Cardinality
@@ -189,7 +193,8 @@ impl ExprVisitor<bool> for ImpureAnalyzer {
             | expr_node::Type::Tand
             | expr_node::Type::ArrayPositions
             | expr_node::Type::StringToArray
-            | expr_node::Type::Format =>
+            | expr_node::Type::Format
+            | expr_node::Type::ArrayTransform =>
             // expression output is deterministic(same result for the same input)
             {
                 let x = func_call
diff --git a/src/frontend/src/expr/type_inference/func.rs b/src/frontend/src/expr/type_inference/func.rs
index 58f5115876813..1febc46788af5 100644
--- a/src/frontend/src/expr/type_inference/func.rs
+++ b/src/frontend/src/expr/type_inference/func.rs
@@ -461,6 +461,17 @@ fn infer_type_for_special(
             }
             Ok(Some(DataType::List(Box::new(DataType::Varchar))))
         }
+        ExprType::RegexpReplace => {
+            // regexp_replace(source, pattern, replacement [, start [, N ]] [, flags ])
+            // TODO: Preprocessing?
+            ensure_arity!("regexp_replace", 3 <= | inputs | <= 6);
+            Ok(Some(DataType::Varchar))
+        }
+        ExprType::RegexpCount => {
+            // TODO: Preprocessing?
+            ensure_arity!("regexp_count", 2 <= | inputs | <= 4);
+            Ok(Some(DataType::Int32))
+        }
         ExprType::ArrayCat => {
             ensure_arity!("array_cat", | inputs | == 2);
             let left_type = (!inputs[0].is_untyped()).then(|| inputs[0].return_type());
@@ -584,6 +595,12 @@ fn infer_type_for_special(
 
             Ok(Some(inputs[0].return_type()))
         }
+        ExprType::ArrayMin => {
+            ensure_arity!("array_min", | inputs | == 1);
+            inputs[0].ensure_array_type()?;
+
+            Ok(Some(inputs[0].return_type().as_list().clone()))
+        }
         ExprType::ArrayDims => {
             ensure_arity!("array_dims", | inputs | == 1);
             inputs[0].ensure_array_type()?;
@@ -596,6 +613,12 @@ fn infer_type_for_special(
             }
             Ok(Some(DataType::Varchar))
         }
+        ExprType::ArrayMax => {
+            ensure_arity!("array_max", | inputs | == 1);
+            inputs[0].ensure_array_type()?;
+
+            Ok(Some(inputs[0].return_type().as_list().clone()))
+        }
         ExprType::StringToArray => {
             ensure_arity!("string_to_array", 2 <= | inputs | <= 3);
 
diff --git a/src/frontend/src/expr/utils.rs b/src/frontend/src/expr/utils.rs
index 53864fd83a08f..d07287b08dbe2 100644
--- a/src/frontend/src/expr/utils.rs
+++ b/src/frontend/src/expr/utils.rs
@@ -447,6 +447,7 @@ impl WatermarkAnalyzer {
             ExprImpl::InputRef(inner) => WatermarkDerivation::Watermark(inner.index()),
             ExprImpl::Literal(_) => WatermarkDerivation::Constant,
             ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
+            ExprImpl::FunctionCallWithLambda(inner) => self.visit_function_call(inner.base()),
             ExprImpl::TableFunction(_) => WatermarkDerivation::None,
             ExprImpl::Subquery(_)
             | ExprImpl::AggCall(_)
diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs
index 13940438dfe5d..3a8c701d77432 100644
--- a/src/frontend/src/handler/create_sink.rs
+++ b/src/frontend/src/handler/create_sink.rs
@@ -73,9 +73,17 @@ pub fn gen_sink_plan(
     let (sink_schema_name, sink_table_name) =
         Binder::resolve_schema_qualified_name(db_name, stmt.sink_name.clone())?;
 
+    // Used for debezium's table name
+    let sink_from_table_name;
     let query = match stmt.sink_from {
-        CreateSink::From(from_name) => Box::new(gen_sink_query_from_name(from_name)?),
-        CreateSink::AsQuery(query) => query,
+        CreateSink::From(from_name) => {
+            sink_from_table_name = from_name.0.last().unwrap().real_value();
+            Box::new(gen_sink_query_from_name(from_name)?)
+        }
+        CreateSink::AsQuery(query) => {
+            sink_from_table_name = sink_table_name.clone();
+            query
+        }
     };
 
     let (sink_database_id, sink_schema_id) =
@@ -117,6 +125,8 @@ pub fn gen_sink_plan(
         definition,
         with_options,
         emit_on_window_close,
+        db_name.to_owned(),
+        sink_from_table_name,
     )?;
     let sink_desc = sink_plan.sink_desc().clone();
     let sink_plan: PlanRef = sink_plan.into();
diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs
index 989e41630e4e7..7b38e6289e164 100644
--- a/src/frontend/src/handler/create_source.rs
+++ b/src/frontend/src/handler/create_source.rs
@@ -27,8 +27,8 @@ use risingwave_common::error::ErrorCode::{self, InvalidInputSyntax, ProtocolErro
 use risingwave_common::error::{Result, RwError};
 use risingwave_common::types::DataType;
 use risingwave_connector::parser::{
-    name_strategy_from_str, AvroParserConfig, DebeziumAvroParserConfig, ProtobufParserConfig,
-    SpecificParserConfig,
+    name_strategy_from_str, schema_to_columns, AvroParserConfig, DebeziumAvroParserConfig,
+    ProtobufParserConfig, SpecificParserConfig,
 };
 use risingwave_connector::source::cdc::{
     CITUS_CDC_CONNECTOR, MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR,
@@ -38,7 +38,7 @@ use risingwave_connector::source::filesystem::S3_CONNECTOR;
 use risingwave_connector::source::nexmark::source::{get_event_data_types_with_names, EventType};
 use risingwave_connector::source::{
     SourceEncode, SourceFormat, SourceStruct, GOOGLE_PUBSUB_CONNECTOR, KAFKA_CONNECTOR,
-    KINESIS_CONNECTOR, NEXMARK_CONNECTOR, PULSAR_CONNECTOR,
+    KINESIS_CONNECTOR, NATS_CONNECTOR, NEXMARK_CONNECTOR, PULSAR_CONNECTOR,
 };
 use risingwave_pb::catalog::{
     PbSchemaRegistryNameStrategy, PbSource, StreamSourceInfo, WatermarkDesc,
@@ -66,6 +66,33 @@ use crate::{bind_data_type, WithOptions};
 pub(crate) const UPSTREAM_SOURCE_KEY: &str = "connector";
 pub(crate) const CONNECTION_NAME_KEY: &str = "connection.name";
 
+/// Map a JSON schema to a relational schema
+async fn extract_json_table_schema(
+    schema_config: &Option<(AstString, bool)>,
+    with_properties: &HashMap<String, String>,
+) -> Result<Option<Vec<ColumnCatalog>>> {
+    match schema_config {
+        None => Ok(None),
+        Some((schema_location, use_schema_registry)) => Ok(Some(
+            schema_to_columns(&schema_location.0, *use_schema_registry, with_properties)
+                .await?
+                .into_iter()
+                .map(|col| ColumnCatalog {
+                    column_desc: col.into(),
+                    is_hidden: false,
+                })
+                .collect_vec(),
+        )),
+    }
+}
+
+fn json_schema_infer_use_schema_registry(schema_config: &Option<(AstString, bool)>) -> bool {
+    match schema_config {
+        None => false,
+        Some((_, use_registry)) => *use_registry,
+    }
+}
+
 /// Map an Avro schema to a relational schema.
 async fn extract_avro_table_schema(
     info: &StreamSourceInfo,
@@ -225,6 +252,21 @@ fn consume_string_from_options(
     ))))
 }
 
+fn get_json_schema_location(
+    row_options: &mut BTreeMap<String, String>,
+) -> Result<Option<(AstString, bool)>> {
+    let schema_location = try_consume_string_from_options(row_options, "schema.location");
+    let schema_registry = try_consume_string_from_options(row_options, "schema.registry");
+    match (schema_location, schema_registry) {
+        (None, None) => Ok(None),
+        (None, Some(schema_registry)) => Ok(Some((schema_registry, true))),
+        (Some(schema_location), None) => Ok(Some((schema_location, false))),
+        (Some(_), Some(_)) => Err(RwError::from(ProtocolError(
+            "only need either the schema location or the schema registry".to_string(),
+        ))),
+    }
+}
+
 fn get_schema_location(row_options: &mut BTreeMap<String, String>) -> Result<(AstString, bool)> {
     let schema_location = try_consume_string_from_options(row_options, "schema.location");
     let schema_registry = try_consume_string_from_options(row_options, "schema.registry");
@@ -331,15 +373,29 @@ pub(crate) async fn try_bind_columns_from_source(
                 },
             )
         }
-        (Format::Plain, Encode::Json) => (
-            None,
-            sql_defined_pk_names,
-            StreamSourceInfo {
-                format: FormatType::Plain as i32,
-                row_encode: EncodeType::Json as i32,
-                ..Default::default()
-            },
-        ),
+        (Format::Plain, Encode::Json) => {
+            let schema_config = get_json_schema_location(&mut options)?;
+            if schema_config.is_some() && sql_defined_schema {
+                return Err(RwError::from(ProtocolError(
+                    "User-defined schema is not allowed with schema registry.".to_string(),
+                )));
+            }
+            if schema_config.is_none() && sql_defined_columns.is_empty() {
+                return Err(RwError::from(InvalidInputSyntax(
+                    "schema definition is required for ENCODE JSON".to_owned(),
+                )));
+            }
+            (
+                extract_json_table_schema(&schema_config, with_properties).await?,
+                sql_defined_pk_names,
+                StreamSourceInfo {
+                    format: FormatType::Plain as i32,
+                    row_encode: EncodeType::Json as i32,
+                    use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
+                    ..Default::default()
+                },
+            )
+        }
         (Format::Plain, Encode::Avro) => {
             let (row_schema_location, use_schema_registry) = get_schema_location(&mut options)?;
             let avro_schema = AvroSchema {
@@ -424,12 +480,17 @@ pub(crate) async fn try_bind_columns_from_source(
             )
         }
         (Format::Upsert, Encode::Json) => {
+            let schema_config = get_json_schema_location(&mut options)?;
+            let columns = extract_json_table_schema(&schema_config, with_properties).await?;
             let (columns, pk_names) = if !sql_defined_pk {
-                let mut columns = bind_sql_columns(sql_defined_columns)?;
+                let mut columns = match columns {
+                    None => bind_sql_columns(sql_defined_columns)?,
+                    Some(columns) => columns,
+                };
                 add_upsert_default_key_column(&mut columns);
                 (Some(columns), vec![DEFAULT_KEY_COLUMN_NAME.into()])
             } else {
-                (None, sql_defined_pk_names)
+                (columns, sql_defined_pk_names)
             };
             (
                 columns,
@@ -437,6 +498,7 @@ pub(crate) async fn try_bind_columns_from_source(
                 StreamSourceInfo {
                     format: FormatType::Upsert as i32,
                     row_encode: EncodeType::Json as i32,
+                    use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
                     ..Default::default()
                 },
             )
@@ -505,12 +567,14 @@ pub(crate) async fn try_bind_columns_from_source(
                         .to_string(),
                 )));
             }
+            let schema_config = get_json_schema_location(&mut options)?;
             (
-                None,
+                extract_json_table_schema(&schema_config, with_properties).await?,
                 sql_defined_pk_names,
                 StreamSourceInfo {
                     format: FormatType::Debezium as i32,
                     row_encode: EncodeType::Json as i32,
+                    use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
                     ..Default::default()
                 },
             )
@@ -660,12 +724,14 @@ pub(crate) async fn try_bind_columns_from_source(
     .to_string(),
     )));
             }
+            let schema_config = get_json_schema_location(&mut options)?;
             (
-                None,
+                extract_json_table_schema(&schema_config, with_properties).await?,
                 sql_defined_pk_names,
                 StreamSourceInfo {
                     format: FormatType::Maxwell as i32,
                     row_encode: EncodeType::Json as i32,
+                    use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
                     ..Default::default()
                 },
             )
@@ -678,12 +744,14 @@ pub(crate) async fn try_bind_columns_from_source(
     .to_string(),
     )));
             }
+            let schema_config = get_json_schema_location(&mut options)?;
             (
-                None,
+                extract_json_table_schema(&schema_config, with_properties).await?,
                 sql_defined_pk_names,
                 StreamSourceInfo {
                     format: FormatType::Canal as i32,
                     row_encode: EncodeType::Json as i32,
+                    use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
                     ..Default::default()
                 },
             )
@@ -836,6 +904,9 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
                     Format::Plain => vec![Encode::Bytes],
                     Format::Debezium => vec![Encode::Json],
                 ),
+                NATS_CONNECTOR => hashmap!(
+                    Format::Plain => vec![Encode::Json],
+                ),
         ))
     });
 
@@ -999,11 +1070,6 @@ pub async fn handle_create_source(
 
     let source_schema = stmt.source_schema.into_source_schema_v2();
 
-    if source_schema.row_encode == Encode::Json && stmt.columns.is_empty() {
-        return Err(RwError::from(InvalidInputSyntax(
-            "schema definition is required for ENCODE JSON".to_owned(),
-        )));
-    }
     let mut with_properties = handler_args.with_options.into_inner().into_iter().collect();
     validate_compatibility(&source_schema, &mut with_properties)?;
 
diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs
index ed429a3eec54e..d8419ac98da38 100644
--- a/src/frontend/src/handler/create_table.rs
+++ b/src/frontend/src/handler/create_table.rs
@@ -19,17 +19,20 @@ use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::catalog::{
-    ColumnCatalog, ColumnDesc, TableId, TableVersionId, INITIAL_SOURCE_VERSION_ID,
+    ColumnCatalog, ColumnDesc, TableDesc, TableId, TableVersionId, INITIAL_SOURCE_VERSION_ID,
     INITIAL_TABLE_VERSION_ID, USER_COLUMN_ID_OFFSET,
 };
+use risingwave_common::constants::hummock::TABLE_OPTION_DUMMY_RETENTION_SECOND;
 use risingwave_common::error::{ErrorCode, Result, RwError};
+use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
+use risingwave_connector::source::external::ExternalTableType;
 use risingwave_pb::catalog::source::OptionalAssociatedTableId;
 use risingwave_pb::catalog::{PbSource, PbTable, StreamSourceInfo, WatermarkDesc};
 use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
 use risingwave_pb::plan_common::{DefaultColumnDesc, GeneratedColumnDesc};
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
 use risingwave_sqlparser::ast::{
-    ColumnDef, ColumnOption, DataType as AstDataType, Encode, Format, ObjectName, SourceSchemaV2,
+    ColumnDef, ColumnOption, DataType as AstDataType, Format, ObjectName, SourceSchemaV2,
     SourceWatermark, TableConstraint,
 };
 
@@ -434,7 +437,7 @@ pub(crate) async fn gen_create_table_plan_with_source(
     ensure_table_constraints_supported(&constraints)?;
     let pk_names = bind_pk_names(&column_defs, &constraints)?;
 
-    let (columns_from_resolve_source, pk_names, source_info) =
+    let (columns_from_resolve_source, pk_names, mut source_info) =
         try_bind_columns_from_source(&source_schema, pk_names, &column_defs, &properties).await?;
     let columns_from_sql = bind_sql_columns(&column_defs)?;
 
@@ -469,6 +472,51 @@ pub(crate) async fn gen_create_table_plan_with_source(
         .into());
     }
 
+    let table_type = ExternalTableType::from_properties(&properties);
+    if table_type.can_backfill() && context.session_ctx().config().get_cdc_backfill() {
+        // Add a column for storing the event offset
+        let offset_column = ColumnCatalog::offset_column();
+        let _offset_index = columns.len();
+        columns.push(offset_column);
+
+        const CDC_SNAPSHOT_MODE_KEY: &str = "debezium.snapshot.mode";
+        // debezium connector will only consume changelogs from latest offset on this mode
+        properties.insert(CDC_SNAPSHOT_MODE_KEY.into(), "rw_cdc_backfill".into());
+
+        let pk_column_indices = {
+            let mut id_to_idx = HashMap::new();
+            columns.iter().enumerate().for_each(|(idx, c)| {
+                id_to_idx.insert(c.column_id(), idx);
+            });
+            // pk column id must exist in table columns.
+            pk_column_ids
+                .iter()
+                .map(|c| id_to_idx.get(c).copied().unwrap())
+                .collect_vec()
+        };
+        let table_pk = pk_column_indices
+            .iter()
+            .map(|idx| ColumnOrder::new(*idx, OrderType::ascending()))
+            .collect();
+
+        let upstream_table_desc = TableDesc {
+            table_id: TableId::placeholder(),
+            pk: table_pk,
+            columns: columns.iter().map(|c| c.column_desc.clone()).collect(),
+            distribution_key: pk_column_indices.clone(),
+            stream_key: pk_column_indices,
+            append_only,
+            retention_seconds: TABLE_OPTION_DUMMY_RETENTION_SECOND,
+            value_indices: (0..columns.len()).collect_vec(),
+            read_prefix_len_hint: 0,
+            watermark_columns: Default::default(),
+            versioned: false,
+        };
+        tracing::debug!("upstream table desc: {:?}", upstream_table_desc);
+        // save external table info to `source_info`
+        source_info.upstream_table = Some(upstream_table_desc.to_protobuf());
+    }
+
     gen_table_plan_inner(
         context.into(),
         table_name,
@@ -697,12 +745,6 @@ pub async fn handle_create_table(
         Ok(_) => {}
     };
 
-    if let Some(s) = &source_schema && s.row_encode == Encode::Json && columns.is_empty() {
-        return Err(RwError::from(ErrorCode::InvalidInputSyntax(
-            "schema definition is required for ENCODE JSON".to_owned(),
-        )));
-    }
-
     let (graph, source, table) = {
         let context = OptimizerContext::from_handler_args(handler_args);
         let source_schema = check_create_table_with_source(context.with_options(), source_schema)?;
diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs
index 5a9c1099cb826..c598fdd9caaf7 100644
--- a/src/frontend/src/lib.rs
+++ b/src/frontend/src/lib.rs
@@ -52,7 +52,7 @@ pub use planner::Planner;
 mod scheduler;
 pub mod session;
 mod stream_fragmenter;
-use risingwave_common::config::OverrideConfig;
+use risingwave_common::config::{MetricLevel, OverrideConfig};
 pub use stream_fragmenter::build_graph;
 mod utils;
 pub use utils::{explain_stream_graph, WithOptions};
@@ -129,7 +129,7 @@ pub struct FrontendOpts {
     /// >0 = enable metrics
     #[clap(long, env = "RW_METRICS_LEVEL")]
     #[override_opts(path = server.metrics_level)]
-    pub metrics_level: Option<u32>,
+    pub metrics_level: Option<MetricLevel>,
 
     #[clap(long, env = "RW_ENABLE_BARRIER_READ")]
     #[override_opts(path = batch.enable_barrier_read)]
diff --git a/src/frontend/src/meta_client.rs b/src/frontend/src/meta_client.rs
index 9423f2e8e63aa..0ab4fe6d5993b 100644
--- a/src/frontend/src/meta_client.rs
+++ b/src/frontend/src/meta_client.rs
@@ -16,6 +16,7 @@ use std::collections::HashMap;
 
 use risingwave_common::system_param::reader::SystemParamsReader;
 use risingwave_pb::backup_service::MetaSnapshotMetadata;
+use risingwave_pb::catalog::Table;
 use risingwave_pb::ddl_service::DdlProgress;
 use risingwave_pb::hummock::HummockSnapshot;
 use risingwave_pb::meta::cancel_creating_jobs_request::PbJobs;
@@ -67,6 +68,8 @@ pub trait FrontendMetaClient: Send + Sync {
     ) -> Result<Option<SystemParamsReader>>;
 
     async fn list_ddl_progress(&self) -> Result<Vec<DdlProgress>>;
+
+    async fn get_tables(&self, table_ids: &[u32]) -> Result<HashMap<u32, Table>>;
 }
 
 pub struct FrontendMetaClientImpl(pub MetaClient);
@@ -137,4 +140,9 @@ impl FrontendMetaClient for FrontendMetaClientImpl {
         let ddl_progress = self.0.get_ddl_progress().await?;
         Ok(ddl_progress)
     }
+
+    async fn get_tables(&self, table_ids: &[u32]) -> Result<HashMap<u32, Table>> {
+        let tables = self.0.get_tables(table_ids).await?;
+        Ok(tables)
+    }
 }
diff --git a/src/frontend/src/optimizer/logical_optimization.rs b/src/frontend/src/optimizer/logical_optimization.rs
index e05a62934d18b..4d958d21ec044 100644
--- a/src/frontend/src/optimizer/logical_optimization.rs
+++ b/src/frontend/src/optimizer/logical_optimization.rs
@@ -362,8 +362,11 @@ static SET_OPERATION_TO_JOIN: LazyLock<OptimizationStage> = LazyLock::new(|| {
 static GROUPING_SETS: LazyLock<OptimizationStage> = LazyLock::new(|| {
     OptimizationStage::new(
         "Grouping Sets",
-        vec![GroupingSetsToExpandRule::create()],
-        ApplyOrder::BottomUp,
+        vec![
+            GroupingSetsToExpandRule::create(),
+            ExpandToProjectRule::create(),
+        ],
+        ApplyOrder::TopDown,
     )
 });
 
diff --git a/src/frontend/src/optimizer/mod.rs b/src/frontend/src/optimizer/mod.rs
index f732d34a62eba..8daa0e7a45b31 100644
--- a/src/frontend/src/optimizer/mod.rs
+++ b/src/frontend/src/optimizer/mod.rs
@@ -554,12 +554,16 @@ impl PlanRoot {
         definition: String,
         properties: WithOptions,
         emit_on_window_close: bool,
+        db_name: String,
+        sink_from_table_name: String,
     ) -> Result<StreamSink> {
         let stream_plan = self.gen_optimized_stream_plan(emit_on_window_close)?;
 
         StreamSink::create(
             stream_plan,
             sink_name,
+            db_name,
+            sink_from_table_name,
             self.required_dist.clone(),
             self.required_order.clone(),
             self.out_fields.clone(),
diff --git a/src/frontend/src/optimizer/plan_expr_rewriter/const_eval_rewriter.rs b/src/frontend/src/optimizer/plan_expr_rewriter/const_eval_rewriter.rs
index 7a40bb3ee7307..90840a47547e6 100644
--- a/src/frontend/src/optimizer/plan_expr_rewriter/const_eval_rewriter.rs
+++ b/src/frontend/src/optimizer/plan_expr_rewriter/const_eval_rewriter.rs
@@ -37,6 +37,7 @@ impl ExprRewriter for ConstEvalRewriter {
                 ExprImpl::InputRef(inner) => self.rewrite_input_ref(*inner),
                 ExprImpl::Literal(inner) => self.rewrite_literal(*inner),
                 ExprImpl::FunctionCall(inner) => self.rewrite_function_call(*inner),
+                ExprImpl::FunctionCallWithLambda(inner) => self.rewrite_function_call_with_lambda(*inner),
                 ExprImpl::AggCall(inner) => self.rewrite_agg_call(*inner),
                 ExprImpl::Subquery(inner) => self.rewrite_subquery(*inner),
                 ExprImpl::CorrelatedInputRef(inner) => self.rewrite_correlated_input_ref(*inner),
diff --git a/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs b/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs
index 804f0dd02b2ee..3bddc479910c2 100644
--- a/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs
+++ b/src/frontend/src/optimizer/plan_expr_visitor/expr_counter.rs
@@ -41,6 +41,7 @@ impl ExprVisitor<()> for CseExprCounter {
             ExprImpl::InputRef(inner) => self.visit_input_ref(inner),
             ExprImpl::Literal(inner) => self.visit_literal(inner),
             ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
+            ExprImpl::FunctionCallWithLambda(inner) => self.visit_function_call_with_lambda(inner),
             ExprImpl::AggCall(inner) => self.visit_agg_call(inner),
             ExprImpl::Subquery(inner) => self.visit_subquery(inner),
             ExprImpl::CorrelatedInputRef(inner) => self.visit_correlated_input_ref(inner),
diff --git a/src/frontend/src/optimizer/plan_node/batch_delete.rs b/src/frontend/src/optimizer/plan_node/batch_delete.rs
index 4f2c5e4f0a315..600ec6827e3eb 100644
--- a/src/frontend/src/optimizer/plan_node/batch_delete.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_delete.rs
@@ -23,7 +23,7 @@ use super::{
 use crate::optimizer::plan_node::ToLocalBatch;
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
 
-/// `BatchDelete` implements [`LogicalDelete`]
+/// `BatchDelete` implements [`super::LogicalDelete`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct BatchDelete {
     pub base: PlanBase,
diff --git a/src/frontend/src/optimizer/plan_node/batch_insert.rs b/src/frontend/src/optimizer/plan_node/batch_insert.rs
index e278ce3a0f9fa..305de0e2f6eaa 100644
--- a/src/frontend/src/optimizer/plan_node/batch_insert.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_insert.rs
@@ -24,7 +24,7 @@ use crate::expr::Expr;
 use crate::optimizer::plan_node::{PlanBase, ToLocalBatch};
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
 
-/// `BatchInsert` implements [`LogicalInsert`]
+/// `BatchInsert` implements [`super::LogicalInsert`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct BatchInsert {
     pub base: PlanBase,
diff --git a/src/frontend/src/optimizer/plan_node/batch_update.rs b/src/frontend/src/optimizer/plan_node/batch_update.rs
index 782c6ef922251..19bb60b9aa1d8 100644
--- a/src/frontend/src/optimizer/plan_node/batch_update.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_update.rs
@@ -26,7 +26,7 @@ use crate::expr::{Expr, ExprRewriter};
 use crate::optimizer::plan_node::ToLocalBatch;
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
 
-/// `BatchUpdate` implements [`LogicalUpdate`]
+/// `BatchUpdate` implements [`super::LogicalUpdate`]
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct BatchUpdate {
     pub base: PlanBase,
diff --git a/src/frontend/src/optimizer/plan_node/generic/agg.rs b/src/frontend/src/optimizer/plan_node/generic/agg.rs
index d2944f50e5559..63bde692af617 100644
--- a/src/frontend/src/optimizer/plan_node/generic/agg.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/agg.rs
@@ -588,8 +588,9 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
 
 impl_distill_unit_from_fields!(Agg, stream::StreamPlanRef);
 
-/// Rewritten version of [`AggCall`] which uses `InputRef` instead of `ExprImpl`.
-/// Refer to [`LogicalAggBuilder::try_rewrite_agg_call`] for more details.
+/// Rewritten version of [`crate::expr::AggCall`] which uses `InputRef` instead of `ExprImpl`.
+/// Refer to [`crate::optimizer::plan_node::logical_agg::LogicalAggBuilder::try_rewrite_agg_call`]
+/// for more details.
 #[derive(Clone, PartialEq, Eq, Hash)]
 pub struct PlanAggCall {
     /// Kind of aggregation function
diff --git a/src/frontend/src/optimizer/plan_node/generic/insert.rs b/src/frontend/src/optimizer/plan_node/generic/insert.rs
index 92ca976b4ea01..c5bfeb725ff83 100644
--- a/src/frontend/src/optimizer/plan_node/generic/insert.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/insert.rs
@@ -114,7 +114,7 @@ impl<PlanRef: GenericPlanRef> Insert<PlanRef> {
 }
 
 impl<PlanRef: Eq + Hash> Insert<PlanRef> {
-    /// Create a [`LogicalInsert`] node. Used internally by optimizer.
+    /// Create a [`Insert`] node. Used internally by optimizer.
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         input: PlanRef,
diff --git a/src/frontend/src/optimizer/plan_node/generic/over_window.rs b/src/frontend/src/optimizer/plan_node/generic/over_window.rs
index c73c81e9952ed..5f7b0705fba26 100644
--- a/src/frontend/src/optimizer/plan_node/generic/over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/over_window.rs
@@ -28,7 +28,8 @@ use crate::optimizer::property::FunctionalDependencySet;
 use crate::utils::ColIndexMappingRewriteExt;
 use crate::OptimizerContextRef;
 
-/// Rewritten version of [`WindowFunction`] which uses `InputRef` instead of `ExprImpl`.
+/// Rewritten version of [`crate::expr::WindowFunction`] which uses `InputRef` instead of
+/// `ExprImpl`.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct PlanWindowFunction {
     pub kind: WindowFuncKind,
diff --git a/src/frontend/src/optimizer/plan_node/generic/project_set.rs b/src/frontend/src/optimizer/plan_node/generic/project_set.rs
index 75067a89b91e7..e159927f7bfda 100644
--- a/src/frontend/src/optimizer/plan_node/generic/project_set.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/project_set.rs
@@ -30,8 +30,8 @@ use crate::utils::{ColIndexMapping, ColIndexMappingRewriteExt};
 /// See also [`ProjectSetSelectItem`](risingwave_pb::expr::ProjectSetSelectItem) for examples.
 ///
 /// To have a pk, it has a hidden column `projected_row_id` at the beginning. The implementation of
-/// `LogicalProjectSet` is highly similar to [`LogicalProject`], except for the additional hidden
-/// column.
+/// `LogicalProjectSet` is highly similar to [`super::super::LogicalProject`], except for the
+/// additional hidden column.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ProjectSet<PlanRef> {
     pub select_list: Vec<ExprImpl>,
diff --git a/src/frontend/src/optimizer/plan_node/generic/scan.rs b/src/frontend/src/optimizer/plan_node/generic/scan.rs
index fca2fe4a28e25..526a677e279b5 100644
--- a/src/frontend/src/optimizer/plan_node/generic/scan.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/scan.rs
@@ -230,7 +230,7 @@ impl Scan {
     /// Create a `LogicalScan` node. Used internally by optimizer.
     #[allow(clippy::too_many_arguments)]
     pub(crate) fn new(
-        table_name: String, // explain-only
+        table_name: String,
         is_sys_table: bool,
         output_col_idx: Vec<usize>, // the column index in the table
         table_desc: Rc<TableDesc>,
diff --git a/src/frontend/src/optimizer/plan_node/generic/top_n.rs b/src/frontend/src/optimizer/plan_node/generic/top_n.rs
index a530c28362e18..990e37219d19b 100644
--- a/src/frontend/src/optimizer/plan_node/generic/top_n.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/top_n.rs
@@ -36,7 +36,8 @@ pub struct TopN<PlanRef> {
 }
 
 impl<PlanRef: stream::StreamPlanRef> TopN<PlanRef> {
-    /// Infers the state table catalog for [`StreamTopN`] and [`StreamGroupTopN`].
+    /// Infers the state table catalog for [`super::super::StreamTopN`] and
+    /// [`super::super::StreamGroupTopN`].
     pub fn infer_internal_table_catalog(
         &self,
         schema: &Schema,
@@ -194,7 +195,7 @@ impl<PlanRef: GenericPlanRef> GenericPlanNode for TopN<PlanRef> {
     }
 }
 
-/// [`Limit`] is used to specify the number of records to return.
+/// `Limit` is used to specify the number of records to return.
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub enum TopNLimit {
     /// The number of records returned is exactly the same as the number after `LIMIT` in the SQL
@@ -229,7 +230,7 @@ impl TopNLimit {
         }
     }
 
-    /// Whether this [`Limit`] returns at most one record for each value. Only `LIMIT 1` without
+    /// Whether this `Limit` returns at most one record for each value. Only `LIMIT 1` without
     /// `WITH TIES` satisfies this condition.
     pub fn max_one_row(&self) -> bool {
         match self {
diff --git a/src/frontend/src/optimizer/plan_node/logical_agg.rs b/src/frontend/src/optimizer/plan_node/logical_agg.rs
index e9af50603e6cf..4387c8f4f89f3 100644
--- a/src/frontend/src/optimizer/plan_node/logical_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_agg.rs
@@ -260,7 +260,7 @@ impl LogicalAgg {
 /// build the plan like `LogicalAgg - LogicalProject`.
 /// it is constructed by `group_exprs` and collect and rewrite the expression in selection and
 /// having clause.
-struct LogicalAggBuilder {
+pub struct LogicalAggBuilder {
     /// the builder of the input Project
     input_proj_builder: ProjectBuilder,
     /// the group key column indices in the project's output
diff --git a/src/frontend/src/optimizer/plan_node/stream.rs b/src/frontend/src/optimizer/plan_node/stream.rs
index 9600fb0cf611c..e9de735718a07 100644
--- a/src/frontend/src/optimizer/plan_node/stream.rs
+++ b/src/frontend/src/optimizer/plan_node/stream.rs
@@ -711,14 +711,8 @@ pub fn to_stream_prost_body(
                 table: Some(me.table.to_internal_table_prost()),
             })
         }
-        Node::ProjectSet(me) => {
-            let me = &me.core;
-            let select_list = me
-                .select_list
-                .iter()
-                .map(ExprImpl::to_project_set_select_item_proto)
-                .collect();
-            PbNodeBody::ProjectSet(ProjectSetNode { select_list })
+        Node::ProjectSet(_) => {
+            unreachable!()
         }
         Node::Project(me) => PbNodeBody::Project(ProjectNode {
             select_list: me.core.exprs.iter().map(|x| x.to_expr_proto()).collect(),
diff --git a/src/frontend/src/optimizer/plan_node/stream_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_over_window.rs
index 360583c849da9..b07c75a1f261e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_over_window.rs
@@ -58,22 +58,19 @@ impl StreamOverWindow {
 
         let mut order_cols = HashSet::new();
         for idx in self.logical.partition_key_indices() {
-            if !order_cols.contains(&idx) {
+            if order_cols.insert(idx) {
                 tbl_builder.add_order_column(idx, OrderType::ascending());
-                order_cols.insert(idx);
             }
         }
         let read_prefix_len_hint = tbl_builder.get_current_pk_len();
         for o in self.logical.order_key() {
-            if !order_cols.contains(&o.column_index) {
+            if order_cols.insert(o.column_index) {
                 tbl_builder.add_order_column(o.column_index, o.order_type);
-                order_cols.insert(o.column_index);
             }
         }
-        for idx in self.logical.input.logical_pk() {
-            if !order_cols.contains(idx) {
-                tbl_builder.add_order_column(*idx, OrderType::ascending());
-                order_cols.insert(*idx);
+        for &idx in self.logical.input.logical_pk() {
+            if order_cols.insert(idx) {
+                tbl_builder.add_order_column(idx, OrderType::ascending());
             }
         }
 
@@ -123,12 +120,19 @@ impl StreamNode for StreamOverWindow {
             .infer_state_table()
             .with_id(state.gen_table_id_wrapped())
             .to_internal_table_prost();
+        let cache_policy = self
+            .base
+            .ctx
+            .session_ctx()
+            .config()
+            .get_streaming_over_window_cache_policy();
 
         PbNodeBody::OverWindow(OverWindowNode {
             calls,
             partition_by,
             order_by,
             state_table: Some(state_table),
+            cache_policy: cache_policy.to_protobuf() as _,
         })
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/stream_project_set.rs b/src/frontend/src/optimizer/plan_node/stream_project_set.rs
index 986eb97826bf7..619fec1f80d15 100644
--- a/src/frontend/src/optimizer/plan_node/stream_project_set.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_project_set.rs
@@ -28,6 +28,12 @@ use crate::utils::ColIndexMappingRewriteExt;
 pub struct StreamProjectSet {
     pub base: PlanBase,
     logical: generic::ProjectSet<PlanRef>,
+    /// All the watermark derivations, (input_column_idx, expr_idx). And the
+    /// derivation expression is the project_set's expression itself.
+    watermark_derivations: Vec<(usize, usize)>,
+    /// Nondecreasing expression indices. `ProjectSet` can produce watermarks for these
+    /// expressions.
+    nondecreasing_exprs: Vec<usize>,
 }
 
 impl StreamProjectSet {
@@ -37,15 +43,26 @@ impl StreamProjectSet {
             .i2o_col_mapping()
             .rewrite_provided_distribution(input.distribution());
 
+        let mut watermark_derivations = vec![];
+        let mut nondecreasing_exprs = vec![];
         let mut watermark_columns = FixedBitSet::with_capacity(logical.output_len());
         for (expr_idx, expr) in logical.select_list.iter().enumerate() {
-            if let WatermarkDerivation::Watermark(input_idx) = try_derive_watermark(expr) {
-                if input.watermark_columns().contains(input_idx) {
-                    // The first column of ProjectSet is `projected_row_id`.
+            match try_derive_watermark(expr) {
+                WatermarkDerivation::Watermark(input_idx) => {
+                    if input.watermark_columns().contains(input_idx) {
+                        watermark_derivations.push((input_idx, expr_idx));
+                        watermark_columns.insert(expr_idx + 1);
+                    }
+                }
+                WatermarkDerivation::Nondecreasing => {
+                    nondecreasing_exprs.push(expr_idx);
                     watermark_columns.insert(expr_idx + 1);
                 }
+                WatermarkDerivation::Constant => {
+                    // XXX(rc): we can produce one watermark on each recovery for this case.
+                }
+                WatermarkDerivation::None => {}
             }
-            // XXX(rc): do we need to handle `WatermarkDerivation::Nondecreasing` here?
         }
 
         // ProjectSet executor won't change the append-only behavior of the stream, so it depends on
@@ -57,7 +74,12 @@ impl StreamProjectSet {
             input.emit_on_window_close(),
             watermark_columns,
         );
-        StreamProjectSet { base, logical }
+        StreamProjectSet {
+            base,
+            logical,
+            watermark_derivations,
+            nondecreasing_exprs,
+        }
     }
 }
 impl_distill_by_unit!(StreamProjectSet, logical, "StreamProjectSet");
@@ -77,6 +99,11 @@ impl PlanTreeNodeUnary for StreamProjectSet {
 
 impl StreamNode for StreamProjectSet {
     fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody {
+        let (watermark_input_cols, watermark_expr_indices) = self
+            .watermark_derivations
+            .iter()
+            .map(|(i, o)| (*i as u32, *o as u32))
+            .unzip();
         PbNodeBody::ProjectSet(ProjectSetNode {
             select_list: self
                 .logical
@@ -84,6 +111,9 @@ impl StreamNode for StreamProjectSet {
                 .iter()
                 .map(|select_item| select_item.to_project_set_select_item_proto())
                 .collect_vec(),
+            watermark_input_cols,
+            watermark_expr_indices,
+            nondecreasing_exprs: self.nondecreasing_exprs.iter().map(|i| *i as _).collect(),
         })
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/stream_sink.rs b/src/frontend/src/optimizer/plan_node/stream_sink.rs
index 57d50a8987ecd..60e89abf3f5c9 100644
--- a/src/frontend/src/optimizer/plan_node/stream_sink.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_sink.rs
@@ -70,6 +70,8 @@ impl StreamSink {
     pub fn create(
         input: PlanRef,
         name: String,
+        db_name: String,
+        sink_from_table_name: String,
         user_distributed_by: RequiredDist,
         user_order_by: Order,
         user_cols: FixedBitSet,
@@ -82,6 +84,8 @@ impl StreamSink {
             input,
             user_distributed_by,
             name,
+            db_name,
+            sink_from_table_name,
             user_order_by,
             columns,
             definition,
@@ -95,6 +99,8 @@ impl StreamSink {
         input: PlanRef,
         user_distributed_by: RequiredDist,
         name: String,
+        db_name: String,
+        sink_from_name: String,
         user_order_by: Order,
         columns: Vec<ColumnCatalog>,
         definition: String,
@@ -142,6 +148,8 @@ impl StreamSink {
         let sink_desc = SinkDesc {
             id: SinkId::placeholder(),
             name,
+            db_name,
+            sink_from_name,
             definition,
             columns,
             plan_pk: pk,
diff --git a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
index 15d4185bab700..51f61e0f663b8 100644
--- a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
@@ -28,6 +28,7 @@ use super::utils::{childless_record, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanNodeId, PlanRef, StreamNode};
 use crate::catalog::ColumnId;
 use crate::expr::{ExprRewriter, FunctionCall};
+use crate::optimizer::plan_node::generic::GenericPlanRef;
 use crate::optimizer::plan_node::utils::{IndicesDisplay, TableCatalogBuilder};
 use crate::optimizer::property::{Distribution, DistributionDisplay};
 use crate::stream_fragmenter::BuildFragmentGraphState;
@@ -244,6 +245,7 @@ impl StreamTableScan {
             })
             .collect_vec();
 
+        // TODO: snapshot read of upstream mview
         let batch_plan_node = BatchPlanNode {
             table_desc: Some(self.logical.table_desc.to_protobuf()),
             column_ids: upstream_column_ids.clone(),
@@ -283,6 +285,12 @@ impl StreamTableScan {
                 // The table desc used by backfill executor
                 table_desc: Some(self.logical.table_desc.to_protobuf()),
                 state_table: Some(catalog),
+                rate_limit: self
+                    .base
+                    .ctx()
+                    .session_ctx()
+                    .config()
+                    .get_streaming_rate_limit(),
             })),
             stream_key,
             operator_id: self.base.id.0 as u64,
diff --git a/src/frontend/src/optimizer/rule/expand_to_project_rule.rs b/src/frontend/src/optimizer/rule/expand_to_project_rule.rs
new file mode 100644
index 0000000000000..1ed1da0037aba
--- /dev/null
+++ b/src/frontend/src/optimizer/rule/expand_to_project_rule.rs
@@ -0,0 +1,60 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::super::plan_node::*;
+use super::{BoxedRule, Rule};
+use crate::expr::{ExprImpl, InputRef};
+use crate::optimizer::plan_node::generic::GenericPlanRef;
+
+pub struct ExpandToProjectRule {}
+
+impl ExpandToProjectRule {
+    pub fn create() -> BoxedRule {
+        Box::new(Self {})
+    }
+}
+impl Rule for ExpandToProjectRule {
+    fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
+        let expand: &LogicalExpand = plan.as_logical_expand()?;
+        let (input, column_subsets) = expand.clone().decompose();
+        assert!(!column_subsets.is_empty());
+        if column_subsets.len() > 1 {
+            return None;
+        }
+        assert!(column_subsets.len() == 1);
+        let column_subset = column_subsets.get(0).unwrap();
+
+        // if `column_subsets` len equals 1, convert it into a project
+        let mut exprs = Vec::with_capacity(expand.base.schema.len());
+        // Add original input column first
+        for i in 0..input.schema().len() {
+            exprs.push(ExprImpl::InputRef(
+                InputRef::new(i, input.schema()[i].data_type()).into(),
+            ));
+        }
+        // Add column sets
+        for i in 0..input.schema().len() {
+            let expr = if column_subset.contains(&i) {
+                ExprImpl::InputRef(InputRef::new(i, input.schema()[i].data_type()).into())
+            } else {
+                ExprImpl::literal_null(input.schema()[i].data_type())
+            };
+            exprs.push(expr);
+        }
+        // Add flag
+        exprs.push(ExprImpl::literal_bigint(0));
+
+        Some(LogicalProject::create(input, exprs))
+    }
+}
diff --git a/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs b/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs
index 0617a53dc6e5b..2073743c90c17 100644
--- a/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs
+++ b/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs
@@ -80,9 +80,6 @@ impl Rule for GroupingSetsToExpandRule {
         let flag_col_idx = group_keys.len();
         let input_schema_len = input.schema().len();
 
-        // TODO: support GROUPING expression.
-        // TODO: optimize the case existing only one set.
-
         let column_subset = grouping_sets
             .iter()
             .map(|set| set.indices().collect_vec())
diff --git a/src/frontend/src/optimizer/rule/mod.rs b/src/frontend/src/optimizer/rule/mod.rs
index 6e7445a1a8ff2..6542ac8e163bb 100644
--- a/src/frontend/src/optimizer/rule/mod.rs
+++ b/src/frontend/src/optimizer/rule/mod.rs
@@ -142,6 +142,8 @@ mod apply_over_window_transpose_rule;
 pub use apply_over_window_transpose_rule::*;
 mod apply_expand_transpose_rule;
 pub use apply_expand_transpose_rule::*;
+mod expand_to_project_rule;
+pub use expand_to_project_rule::*;
 
 #[macro_export]
 macro_rules! for_all_rules {
@@ -203,6 +205,7 @@ macro_rules! for_all_rules {
             , { BatchProjectMergeRule }
             , { ApplyOverWindowTransposeRule }
             , { ApplyExpandTransposeRule }
+            , { ExpandToProjectRule }
         }
     };
 }
diff --git a/src/frontend/src/scheduler/worker_node_manager.rs b/src/frontend/src/scheduler/worker_node_manager.rs
index 9fb25f0074899..ed1ce5d2aaa6b 100644
--- a/src/frontend/src/scheduler/worker_node_manager.rs
+++ b/src/frontend/src/scheduler/worker_node_manager.rs
@@ -373,11 +373,10 @@ impl WorkerNodeSelector {
     }
 
     fn apply_worker_node_mask(&self, origin: Vec<WorkerNode>) -> Vec<WorkerNode> {
-        if origin.len() <= 1 {
-            // If there is at most one worker, don't apply mask.
+        let mask = self.manager.worker_node_mask();
+        if origin.iter().all(|w| mask.contains(&w.id)) {
             return origin;
         }
-        let mask = self.manager.worker_node_mask();
         origin
             .into_iter()
             .filter(|w| !mask.contains(&w.id))
diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs
index 924ff54608cd8..f4783f0f5a22a 100644
--- a/src/frontend/src/session.rs
+++ b/src/frontend/src/session.rs
@@ -32,7 +32,7 @@ use risingwave_common::catalog::DEFAULT_SCHEMA_NAME;
 use risingwave_common::catalog::{
     DEFAULT_DATABASE_NAME, DEFAULT_SUPER_USER, DEFAULT_SUPER_USER_ID,
 };
-use risingwave_common::config::{load_config, BatchConfig, MetaConfig};
+use risingwave_common::config::{load_config, BatchConfig, MetaConfig, MetricLevel};
 use risingwave_common::error::{ErrorCode, Result, RwError};
 use risingwave_common::session_config::{ConfigMap, ConfigReporter, VisibilityMode};
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
@@ -277,7 +277,7 @@ impl FrontendEnv {
         let frontend_metrics = Arc::new(GLOBAL_FRONTEND_METRICS.clone());
         let source_metrics = Arc::new(GLOBAL_SOURCE_METRICS.clone());
 
-        if config.server.metrics_level > 0 {
+        if config.server.metrics_level > MetricLevel::Disabled {
             MetricsManager::boot_metrics_service(opts.prometheus_listener_addr.clone());
         }
 
diff --git a/src/frontend/src/test_utils.rs b/src/frontend/src/test_utils.rs
index 5f7151d689406..e934bed502f42 100644
--- a/src/frontend/src/test_utils.rs
+++ b/src/frontend/src/test_utils.rs
@@ -32,7 +32,7 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_pb::backup_service::MetaSnapshotMetadata;
 use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
 use risingwave_pb::catalog::{
-    PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView,
+    PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbTable, PbView, Table,
 };
 use risingwave_pb::ddl_service::{create_connection_request, DdlProgress};
 use risingwave_pb::hummock::HummockSnapshot;
@@ -819,6 +819,10 @@ impl FrontendMetaClient for MockFrontendMetaClient {
     async fn list_ddl_progress(&self) -> RpcResult<Vec<DdlProgress>> {
         Ok(vec![])
     }
+
+    async fn get_tables(&self, _table_ids: &[u32]) -> RpcResult<HashMap<u32, Table>> {
+        Ok(HashMap::new())
+    }
 }
 
 #[cfg(test)]
diff --git a/src/java_binding/Cargo.toml b/src/java_binding/Cargo.toml
index 9eda6a43e5bb2..3280125f3ac49 100644
--- a/src/java_binding/Cargo.toml
+++ b/src/java_binding/Cargo.toml
@@ -10,29 +10,12 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
-bytes = "1"
-futures = { version = "0.3", default-features = false, features = ["alloc"] }
-itertools = "0.11"
-jni = "0.21.1"
 prost = "0.11"
 risingwave_common = { workspace = true }
-risingwave_hummock_sdk = { workspace = true }
-risingwave_object_store = { workspace = true }
+risingwave_jni_core = { workspace = true }
 risingwave_pb = { workspace = true }
-risingwave_storage = { workspace = true }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-thiserror = "1"
-tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "fs",
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-] }
-tracing = "0.1"
 
 [dev-dependencies]
 risingwave_expr = { workspace = true }
diff --git a/src/java_binding/src/lib.rs b/src/java_binding/src/lib.rs
index e99137f0a21fe..12a3c59fc829f 100644
--- a/src/java_binding/src/lib.rs
+++ b/src/java_binding/src/lib.rs
@@ -17,820 +17,4 @@
 #![feature(once_cell_try)]
 #![feature(type_alias_impl_trait)]
 
-mod hummock_iterator;
-mod stream_chunk_iterator;
-
-use std::backtrace::Backtrace;
-use std::marker::PhantomData;
-use std::ops::{Deref, DerefMut};
-use std::slice::from_raw_parts;
-use std::sync::{Arc, LazyLock, OnceLock};
-
-use hummock_iterator::{HummockJavaBindingIterator, KeyedRow};
-use jni::objects::{
-    AutoElements, GlobalRef, JByteArray, JClass, JMethodID, JObject, JStaticMethodID, JString,
-    JValue, JValueGen, JValueOwned, ReleaseMode,
-};
-use jni::signature::ReturnType;
-use jni::sys::{jboolean, jbyte, jdouble, jfloat, jint, jlong, jshort, jsize, jvalue};
-use jni::JNIEnv;
-use prost::{DecodeError, Message};
-use risingwave_common::array::{ArrayError, StreamChunk};
-use risingwave_common::hash::VirtualNode;
-use risingwave_common::row::{OwnedRow, Row};
-use risingwave_common::test_prelude::StreamChunkTestExt;
-use risingwave_common::types::ScalarRefImpl;
-use risingwave_common::util::panic::rw_catch_unwind;
-use risingwave_storage::error::StorageError;
-use thiserror::Error;
-use tokio::runtime::Runtime;
-
-use crate::stream_chunk_iterator::{StreamChunkIterator, StreamChunkRow};
-
-static RUNTIME: LazyLock<Runtime> = LazyLock::new(|| tokio::runtime::Runtime::new().unwrap());
-
-#[derive(Error, Debug)]
-enum BindingError {
-    #[error("JniError {error}")]
-    Jni {
-        #[from]
-        error: jni::errors::Error,
-        backtrace: Backtrace,
-    },
-
-    #[error("StorageError {error}")]
-    Storage {
-        #[from]
-        error: StorageError,
-        backtrace: Backtrace,
-    },
-
-    #[error("DecodeError {error}")]
-    Decode {
-        #[from]
-        error: DecodeError,
-        backtrace: Backtrace,
-    },
-
-    #[error("StreamChunkArrayError {error}")]
-    StreamChunkArray {
-        #[from]
-        error: ArrayError,
-        backtrace: Backtrace,
-    },
-}
-
-type Result<T> = std::result::Result<T, BindingError>;
-
-fn to_guarded_slice<'array, 'env>(
-    array: &'array JByteArray<'env>,
-    env: &'array mut JNIEnv<'env>,
-) -> Result<SliceGuard<'env, 'array>> {
-    unsafe {
-        let array = env.get_array_elements(array, ReleaseMode::NoCopyBack)?;
-        let slice = from_raw_parts(array.as_ptr() as *mut u8, array.len());
-
-        Ok(SliceGuard {
-            _array: array,
-            slice,
-        })
-    }
-}
-
-/// Wrapper around `&[u8]` derived from `jbyteArray` to prevent it from being auto-released.
-pub struct SliceGuard<'env, 'array> {
-    _array: AutoElements<'env, 'env, 'array, jbyte>,
-    slice: &'array [u8],
-}
-
-impl<'env, 'array> Deref for SliceGuard<'env, 'array> {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        self.slice
-    }
-}
-
-#[repr(transparent)]
-pub struct Pointer<'a, T> {
-    pointer: jlong,
-    _phantom: PhantomData<&'a T>,
-}
-
-impl<'a, T> Default for Pointer<'a, T> {
-    fn default() -> Self {
-        Self {
-            pointer: 0,
-            _phantom: Default::default(),
-        }
-    }
-}
-
-impl<T> From<T> for Pointer<'static, T> {
-    fn from(value: T) -> Self {
-        Pointer {
-            pointer: Box::into_raw(Box::new(value)) as jlong,
-            _phantom: PhantomData,
-        }
-    }
-}
-
-impl<T> Pointer<'static, T> {
-    fn null() -> Self {
-        Pointer {
-            pointer: 0,
-            _phantom: PhantomData,
-        }
-    }
-}
-
-impl<'a, T> Pointer<'a, T> {
-    fn as_ref(&self) -> &'a T {
-        debug_assert!(self.pointer != 0);
-        unsafe { &*(self.pointer as *const T) }
-    }
-
-    fn as_mut(&mut self) -> &'a mut T {
-        debug_assert!(self.pointer != 0);
-        unsafe { &mut *(self.pointer as *mut T) }
-    }
-
-    fn drop(self) {
-        debug_assert!(self.pointer != 0);
-        unsafe { drop(Box::from_raw(self.pointer as *mut T)) }
-    }
-}
-
-/// In most Jni interfaces, the first parameter is `JNIEnv`, and the second parameter is `JClass`.
-/// This struct simply encapsulates the two common parameters into a single struct for simplicity.
-#[repr(C)]
-pub struct EnvParam<'a> {
-    env: JNIEnv<'a>,
-    class: JClass<'a>,
-}
-
-impl<'a> Deref for EnvParam<'a> {
-    type Target = JNIEnv<'a>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.env
-    }
-}
-
-impl<'a> DerefMut for EnvParam<'a> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.env
-    }
-}
-
-impl<'a> EnvParam<'a> {
-    pub fn get_class(&self) -> &JClass<'a> {
-        &self.class
-    }
-}
-
-fn execute_and_catch<'env, F, Ret>(mut env: EnvParam<'env>, inner: F) -> Ret
-where
-    F: FnOnce(&mut EnvParam<'env>) -> Result<Ret>,
-    Ret: Default + 'env,
-{
-    match rw_catch_unwind(std::panic::AssertUnwindSafe(|| inner(&mut env))) {
-        Ok(Ok(ret)) => ret,
-        Ok(Err(e)) => {
-            match e {
-                BindingError::Jni {
-                    error: jni::errors::Error::JavaException,
-                    backtrace,
-                } => {
-                    tracing::error!("get JavaException thrown from: {:?}", backtrace);
-                    // the exception is already thrown. No need to throw again
-                }
-                _ => {
-                    env.throw(format!("get error while processing: {:?}", e))
-                        .expect("should be able to throw");
-                }
-            }
-            Ret::default()
-        }
-        Err(e) => {
-            env.throw(format!("panic while processing: {:?}", e))
-                .expect("should be able to throw");
-            Ret::default()
-        }
-    }
-}
-
-pub enum JavaBindingRowInner {
-    Keyed(KeyedRow),
-    StreamChunk(StreamChunkRow),
-}
-#[derive(Default)]
-pub struct JavaClassMethodCache {
-    big_decimal_ctor: OnceLock<(GlobalRef, JMethodID)>,
-    timestamp_ctor: OnceLock<(GlobalRef, JMethodID)>,
-
-    date_ctor: OnceLock<(GlobalRef, JStaticMethodID)>,
-    time_ctor: OnceLock<(GlobalRef, JStaticMethodID)>,
-}
-
-pub struct JavaBindingRow {
-    inner: JavaBindingRowInner,
-    class_cache: Arc<JavaClassMethodCache>,
-}
-
-impl JavaBindingRow {
-    fn with_stream_chunk(
-        underlying: StreamChunkRow,
-        class_cache: Arc<JavaClassMethodCache>,
-    ) -> Self {
-        Self {
-            inner: JavaBindingRowInner::StreamChunk(underlying),
-            class_cache,
-        }
-    }
-
-    fn with_keyed(underlying: KeyedRow, class_cache: Arc<JavaClassMethodCache>) -> Self {
-        Self {
-            inner: JavaBindingRowInner::Keyed(underlying),
-            class_cache,
-        }
-    }
-
-    fn as_keyed(&self) -> &KeyedRow {
-        match &self.inner {
-            JavaBindingRowInner::Keyed(r) => r,
-            _ => unreachable!("can only call as_keyed for KeyedRow"),
-        }
-    }
-
-    fn as_stream_chunk(&self) -> &StreamChunkRow {
-        match &self.inner {
-            JavaBindingRowInner::StreamChunk(r) => r,
-            _ => unreachable!("can only call as_stream_chunk for StreamChunkRow"),
-        }
-    }
-}
-
-impl Deref for JavaBindingRow {
-    type Target = OwnedRow;
-
-    fn deref(&self) -> &Self::Target {
-        match &self.inner {
-            JavaBindingRowInner::Keyed(r) => r.row(),
-            JavaBindingRowInner::StreamChunk(r) => r.row(),
-        }
-    }
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_vnodeCount(
-    _env: EnvParam<'_>,
-) -> jint {
-    VirtualNode::COUNT as jint
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_hummockIteratorNew<'a>(
-    env: EnvParam<'a>,
-    read_plan: JByteArray<'a>,
-) -> Pointer<'static, HummockJavaBindingIterator> {
-    execute_and_catch(env, move |env| {
-        let read_plan = Message::decode(to_guarded_slice(&read_plan, env)?.deref())?;
-        let iter = RUNTIME.block_on(HummockJavaBindingIterator::new(read_plan))?;
-        Ok(iter.into())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_hummockIteratorNext<'a>(
-    env: EnvParam<'a>,
-    mut pointer: Pointer<'a, HummockJavaBindingIterator>,
-) -> Pointer<'static, JavaBindingRow> {
-    execute_and_catch(env, move |_env| {
-        let iter = pointer.as_mut();
-        match RUNTIME.block_on(iter.next())? {
-            None => Ok(Pointer::null()),
-            Some(row) => Ok(JavaBindingRow::with_keyed(row, iter.class_cache.clone()).into()),
-        }
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_hummockIteratorClose(
-    _env: EnvParam<'_>,
-    pointer: Pointer<'_, HummockJavaBindingIterator>,
-) {
-    pointer.drop();
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorNew<'a>(
-    env: EnvParam<'a>,
-    stream_chunk_payload: JByteArray<'a>,
-) -> Pointer<'static, StreamChunkIterator> {
-    execute_and_catch(env, move |env| {
-        let prost_stream_chumk =
-            Message::decode(to_guarded_slice(&stream_chunk_payload, env)?.deref())?;
-        let iter = StreamChunkIterator::new(StreamChunk::from_protobuf(&prost_stream_chumk)?);
-        Ok(iter.into())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorFromPretty<
-    'a,
->(
-    env: EnvParam<'a>,
-    str: JString<'a>,
-) -> Pointer<'static, StreamChunkIterator> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let iter = StreamChunkIterator::new(StreamChunk::from_pretty(
-            env.get_string(&str)
-                .expect("cannot get java string")
-                .to_str()
-                .unwrap(),
-        ));
-        Ok(iter.into())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorNext<'a>(
-    env: EnvParam<'a>,
-    mut pointer: Pointer<'a, StreamChunkIterator>,
-) -> Pointer<'static, JavaBindingRow> {
-    execute_and_catch(env, move |_env| {
-        let iter = pointer.as_mut();
-        match iter.next() {
-            None => Ok(Pointer::null()),
-            Some(row) => {
-                Ok(JavaBindingRow::with_stream_chunk(row, iter.class_cache.clone()).into())
-            }
-        }
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorClose(
-    _env: EnvParam<'_>,
-    pointer: Pointer<'_, StreamChunkIterator>,
-) {
-    pointer.drop();
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetKey<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-) -> JByteArray<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        Ok(env.byte_array_from_slice(pointer.as_ref().as_keyed().key())?)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetOp<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-) -> jint {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer.as_ref().as_stream_chunk().op() as jint)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowIsNull<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jboolean {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer.as_ref().datum_at(idx as usize).is_none() as jboolean)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetInt16Value<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jshort {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_int16())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetInt32Value<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jint {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_int32())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetInt64Value<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jlong {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_int64())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetFloatValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jfloat {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_float32()
-            .into())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetDoubleValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jdouble {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_float64()
-            .into())
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetBooleanValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> jboolean {
-    execute_and_catch(env, move |_env| {
-        Ok(pointer.as_ref().datum_at(idx as usize).unwrap().into_bool() as jboolean)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetStringValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JString<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'a>| {
-        Ok(env.new_string(pointer.as_ref().datum_at(idx as usize).unwrap().into_utf8())?)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetIntervalValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JString<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'a>| {
-        let interval = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_interval()
-            .as_iso_8601();
-        Ok(env.new_string(interval)?)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetJsonbValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JString<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let jsonb = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_jsonb()
-            .to_string();
-        Ok(env.new_string(jsonb)?)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetTimestampValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JObject<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let scalar_value = pointer.as_ref().datum_at(idx as usize).unwrap();
-        let millis = match scalar_value {
-            // supports sinking rw timestamptz to mysql timestamp
-            ScalarRefImpl::Timestamptz(tz) => tz.timestamp_millis(),
-            ScalarRefImpl::Timestamp(ts) => ts.0.timestamp_millis(),
-            _ => panic!("expect timestamp or timestamptz"),
-        };
-        let (ts_class_ref, constructor) = pointer
-            .as_ref()
-            .class_cache
-            .timestamp_ctor
-            .get_or_try_init(|| {
-                let cls = env.find_class("java/sql/Timestamp")?;
-                let init_method = env.get_method_id(&cls, "<init>", "(J)V")?;
-                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
-            })?;
-        unsafe {
-            let ts_class = <&JClass<'_>>::from(ts_class_ref.as_obj());
-            let date_obj =
-                env.new_object_unchecked(ts_class, *constructor, &[jvalue { j: millis }])?;
-            Ok(date_obj)
-        }
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetDecimalValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JObject<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let value = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_decimal()
-            .to_string();
-        let string_value = env.new_string(value)?;
-        let (decimal_class_ref, constructor) = pointer
-            .as_ref()
-            .class_cache
-            .big_decimal_ctor
-            .get_or_try_init(|| {
-                let cls = env.find_class("java/math/BigDecimal")?;
-                let init_method = env.get_method_id(&cls, "<init>", "(Ljava/lang/String;)V")?;
-                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
-            })?;
-        unsafe {
-            let decimal_class = <&JClass<'_>>::from(decimal_class_ref.as_obj());
-            let date_obj = env.new_object_unchecked(
-                decimal_class,
-                *constructor,
-                &[jvalue {
-                    l: string_value.into_raw(),
-                }],
-            )?;
-            Ok(date_obj)
-        }
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetDateValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JObject<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let value = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_date()
-            .0
-            .to_string();
-
-        let string_value = env.new_string(value)?;
-        let (class_ref, constructor) =
-            pointer.as_ref().class_cache.date_ctor.get_or_try_init(|| {
-                let cls = env.find_class("java/sql/Date")?;
-                let init_method = env.get_static_method_id(
-                    &cls,
-                    "valueOf",
-                    "(Ljava/lang/String;)Ljava/sql/Date;",
-                )?;
-                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
-            })?;
-        unsafe {
-            let JValueOwned::Object(date_obj) = env.call_static_method_unchecked(
-                <&JClass<'_>>::from(class_ref.as_obj()),
-                *constructor,
-                ReturnType::Object,
-                &[jvalue {
-                    l: string_value.into_raw(),
-                }],
-            )?
-            else {
-                return Err(BindingError::from(jni::errors::Error::MethodNotFound {
-                    name: "valueOf".to_string(),
-                    sig: "(Ljava/lang/String;)Ljava/sql/Date;".into(),
-                }));
-            };
-            Ok(date_obj)
-        }
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetTimeValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JObject<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let value = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_time()
-            .0
-            .to_string();
-
-        let string_value = env.new_string(value)?;
-        let (class_ref, constructor) =
-            pointer.as_ref().class_cache.time_ctor.get_or_try_init(|| {
-                let cls = env.find_class("java/sql/Time")?;
-                let init_method = env.get_static_method_id(
-                    &cls,
-                    "valueOf",
-                    "(Ljava/lang/String;)Ljava/sql/Time;",
-                )?;
-                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
-            })?;
-        unsafe {
-            let class = <&JClass<'_>>::from(class_ref.as_obj());
-            match env.call_static_method_unchecked(
-                class,
-                *constructor,
-                ReturnType::Object,
-                &[jvalue {
-                    l: string_value.into_raw(),
-                }],
-            )? {
-                JValueGen::Object(obj) => Ok(obj),
-                _ => Err(BindingError::from(jni::errors::Error::MethodNotFound {
-                    name: "valueOf".to_string(),
-                    sig: "(Ljava/lang/String;)Ljava/sql/Time;".into(),
-                })),
-            }
-        }
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetByteaValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-) -> JByteArray<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let bytes = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_bytea();
-        Ok(env.byte_array_from_slice(bytes)?)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetArrayValue<'a>(
-    env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-    idx: jint,
-    class: JClass<'a>,
-) -> JObject<'a> {
-    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
-        let elems = pointer
-            .as_ref()
-            .datum_at(idx as usize)
-            .unwrap()
-            .into_list()
-            .iter();
-
-        // convert the Rust elements to a Java object array (Object[])
-        let jarray = env.new_object_array(elems.len() as jsize, &class, JObject::null())?;
-
-        for (i, ele) in elems.enumerate() {
-            let index = i as jsize;
-            match ele {
-                None => env.set_object_array_element(&jarray, i as jsize, JObject::null())?,
-                Some(val) => match val {
-                    ScalarRefImpl::Int16(v) => {
-                        let obj = env.call_static_method(
-                            &class,
-                            "valueOf",
-                            "(S)Ljava.lang.Short;",
-                            &[JValue::from(v as jshort)],
-                        )?;
-                        if let JValueOwned::Object(o) = obj {
-                            env.set_object_array_element(&jarray, index, &o)?
-                        }
-                    }
-                    ScalarRefImpl::Int32(v) => {
-                        let obj = env.call_static_method(
-                            &class,
-                            "valueOf",
-                            "(I)Ljava.lang.Integer;",
-                            &[JValue::from(v as jint)],
-                        )?;
-                        if let JValueOwned::Object(o) = obj {
-                            env.set_object_array_element(&jarray, index, &o)?
-                        }
-                    }
-                    ScalarRefImpl::Int64(v) => {
-                        let obj = env.call_static_method(
-                            &class,
-                            "valueOf",
-                            "(J)Ljava.lang.Long;",
-                            &[JValue::from(v as jlong)],
-                        )?;
-                        if let JValueOwned::Object(o) = obj {
-                            env.set_object_array_element(&jarray, index, &o)?
-                        }
-                    }
-                    ScalarRefImpl::Float32(v) => {
-                        let obj = env.call_static_method(
-                            &class,
-                            "valueOf",
-                            "(F)Ljava/lang/Float;",
-                            &[JValue::from(v.into_inner() as jfloat)],
-                        )?;
-                        if let JValueOwned::Object(o) = obj {
-                            env.set_object_array_element(&jarray, index, &o)?
-                        }
-                    }
-                    ScalarRefImpl::Float64(v) => {
-                        let obj = env.call_static_method(
-                            &class,
-                            "valueOf",
-                            "(D)Ljava/lang/Double;",
-                            &[JValue::from(v.into_inner() as jdouble)],
-                        )?;
-                        if let JValueOwned::Object(o) = obj {
-                            env.set_object_array_element(&jarray, index, &o)?
-                        }
-                    }
-                    ScalarRefImpl::Utf8(v) => {
-                        let obj = env.new_string(v)?;
-                        env.set_object_array_element(&jarray, index, obj)?
-                    }
-                    _ => env.set_object_array_element(&jarray, index, JObject::null())?,
-                },
-            }
-        }
-        let output = unsafe { JObject::from_raw(jarray.into_raw()) };
-        Ok(output)
-    })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowClose<'a>(
-    _env: EnvParam<'a>,
-    pointer: Pointer<'a, JavaBindingRow>,
-) {
-    pointer.drop()
-}
-
-#[cfg(test)]
-mod tests {
-    use risingwave_common::types::{DataType, Timestamptz};
-    use risingwave_expr::vector_op::cast::literal_parsing;
-
-    /// make sure that the [`ScalarRefImpl::Int64`] received by
-    /// [`Java_com_risingwave_java_binding_Binding_rowGetTimestampValue`]
-    /// is of type [`DataType::Timestamptz`] stored in microseconds
-    #[test]
-    fn test_timestamptz_to_i64() {
-        assert_eq!(
-            literal_parsing(&DataType::Timestamptz, "2023-06-01 09:45:00+08:00").unwrap(),
-            Timestamptz::from_micros(1_685_583_900_000_000).into()
-        );
-    }
-}
+pub use risingwave_jni_core::*;
diff --git a/src/jni_core/Cargo.toml b/src/jni_core/Cargo.toml
new file mode 100644
index 0000000000000..e0e4bd75022fd
--- /dev/null
+++ b/src/jni_core/Cargo.toml
@@ -0,0 +1,38 @@
+[package]
+name = "risingwave_jni_core"
+version = "0.1.0"
+edition = "2021"
+
+[package.metadata.cargo-machete]
+ignored = ["workspace-hack"]
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-hack"]
+
+[dependencies]
+bytes = "1"
+futures = { version = "0.3", default-features = false, features = ["alloc"] }
+itertools = "0.11"
+jni = "0.21.1"
+prost = "0.11"
+risingwave_common = { workspace = true }
+risingwave_hummock_sdk = { workspace = true }
+risingwave_object_store = { workspace = true }
+risingwave_pb = { workspace = true }
+risingwave_storage = { workspace = true }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "1"
+tokio = { version = "0.2", package = "madsim-tokio", features = [
+    "fs",
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
+
+[dev-dependencies]
+risingwave_expr = { workspace = true }
diff --git a/src/java_binding/src/hummock_iterator.rs b/src/jni_core/src/hummock_iterator.rs
similarity index 95%
rename from src/java_binding/src/hummock_iterator.rs
rename to src/jni_core/src/hummock_iterator.rs
index 0f21449ec6ea6..92bb09885f960 100644
--- a/src/java_binding/src/hummock_iterator.rs
+++ b/src/jni_core/src/hummock_iterator.rs
@@ -102,13 +102,9 @@ impl HummockJavaBindingIterator {
                     ),
                     read_plan.epoch,
                     ReadOptions {
-                        prefix_hint: None,
-                        ignore_range_tombstone: false,
-                        retention_seconds: None,
                         table_id: read_plan.table_id.into(),
-                        read_version_from_backup: false,
-                        prefetch_options: Default::default(),
                         cache_policy: CachePolicy::NotFill,
+                        ..Default::default()
                     },
                     (vec![], vec![], pin_version.clone()),
                 )
diff --git a/src/jni_core/src/lib.rs b/src/jni_core/src/lib.rs
new file mode 100644
index 0000000000000..62625e14d21fe
--- /dev/null
+++ b/src/jni_core/src/lib.rs
@@ -0,0 +1,841 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(error_generic_member_access)]
+#![feature(provide_any)]
+#![feature(lazy_cell)]
+#![feature(once_cell_try)]
+#![feature(type_alias_impl_trait)]
+#![feature(result_option_inspect)]
+
+pub mod hummock_iterator;
+pub mod stream_chunk_iterator;
+
+use std::backtrace::Backtrace;
+use std::marker::PhantomData;
+use std::ops::{Deref, DerefMut};
+use std::slice::from_raw_parts;
+use std::sync::{Arc, LazyLock, OnceLock};
+
+use hummock_iterator::{HummockJavaBindingIterator, KeyedRow};
+use jni::objects::{
+    AutoElements, GlobalRef, JByteArray, JClass, JMethodID, JObject, JStaticMethodID, JString,
+    JValue, JValueGen, JValueOwned, ReleaseMode,
+};
+use jni::signature::ReturnType;
+use jni::sys::{jboolean, jbyte, jdouble, jfloat, jint, jlong, jshort, jsize, jvalue};
+use jni::JNIEnv;
+use prost::{DecodeError, Message};
+use risingwave_common::array::{ArrayError, StreamChunk};
+use risingwave_common::hash::VirtualNode;
+use risingwave_common::row::{OwnedRow, Row};
+use risingwave_common::test_prelude::StreamChunkTestExt;
+use risingwave_common::types::ScalarRefImpl;
+use risingwave_common::util::panic::rw_catch_unwind;
+use risingwave_pb::connector_service::GetEventStreamResponse;
+use risingwave_storage::error::StorageError;
+use thiserror::Error;
+use tokio::runtime::Runtime;
+use tokio::sync::mpsc::Sender;
+
+use crate::stream_chunk_iterator::{StreamChunkIterator, StreamChunkRow};
+pub type GetEventStreamJniSender = Sender<GetEventStreamResponse>;
+
+static RUNTIME: LazyLock<Runtime> = LazyLock::new(|| tokio::runtime::Runtime::new().unwrap());
+
+#[derive(Error, Debug)]
+enum BindingError {
+    #[error("JniError {error}")]
+    Jni {
+        #[from]
+        error: jni::errors::Error,
+        backtrace: Backtrace,
+    },
+
+    #[error("StorageError {error}")]
+    Storage {
+        #[from]
+        error: StorageError,
+        backtrace: Backtrace,
+    },
+
+    #[error("DecodeError {error}")]
+    Decode {
+        #[from]
+        error: DecodeError,
+        backtrace: Backtrace,
+    },
+
+    #[error("StreamChunkArrayError {error}")]
+    StreamChunkArray {
+        #[from]
+        error: ArrayError,
+        backtrace: Backtrace,
+    },
+}
+
+type Result<T> = std::result::Result<T, BindingError>;
+
+fn to_guarded_slice<'array, 'env>(
+    array: &'array JByteArray<'env>,
+    env: &'array mut JNIEnv<'env>,
+) -> Result<SliceGuard<'env, 'array>> {
+    unsafe {
+        let array = env.get_array_elements(array, ReleaseMode::NoCopyBack)?;
+        let slice = from_raw_parts(array.as_ptr() as *mut u8, array.len());
+
+        Ok(SliceGuard {
+            _array: array,
+            slice,
+        })
+    }
+}
+
+/// Wrapper around `&[u8]` derived from `jbyteArray` to prevent it from being auto-released.
+pub struct SliceGuard<'env, 'array> {
+    _array: AutoElements<'env, 'env, 'array, jbyte>,
+    slice: &'array [u8],
+}
+
+impl<'env, 'array> Deref for SliceGuard<'env, 'array> {
+    type Target = [u8];
+
+    fn deref(&self) -> &Self::Target {
+        self.slice
+    }
+}
+
+#[repr(transparent)]
+pub struct Pointer<'a, T> {
+    pointer: jlong,
+    _phantom: PhantomData<&'a T>,
+}
+
+impl<'a, T> Default for Pointer<'a, T> {
+    fn default() -> Self {
+        Self {
+            pointer: 0,
+            _phantom: Default::default(),
+        }
+    }
+}
+
+impl<T> From<T> for Pointer<'static, T> {
+    fn from(value: T) -> Self {
+        Pointer {
+            pointer: Box::into_raw(Box::new(value)) as jlong,
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<T> Pointer<'static, T> {
+    fn null() -> Self {
+        Pointer {
+            pointer: 0,
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<'a, T> Pointer<'a, T> {
+    fn as_ref(&self) -> &'a T {
+        debug_assert!(self.pointer != 0);
+        unsafe { &*(self.pointer as *const T) }
+    }
+
+    fn as_mut(&mut self) -> &'a mut T {
+        debug_assert!(self.pointer != 0);
+        unsafe { &mut *(self.pointer as *mut T) }
+    }
+
+    fn drop(self) {
+        debug_assert!(self.pointer != 0);
+        unsafe { drop(Box::from_raw(self.pointer as *mut T)) }
+    }
+}
+
+/// In most Jni interfaces, the first parameter is `JNIEnv`, and the second parameter is `JClass`.
+/// This struct simply encapsulates the two common parameters into a single struct for simplicity.
+#[repr(C)]
+pub struct EnvParam<'a> {
+    env: JNIEnv<'a>,
+    class: JClass<'a>,
+}
+
+impl<'a> Deref for EnvParam<'a> {
+    type Target = JNIEnv<'a>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.env
+    }
+}
+
+impl<'a> DerefMut for EnvParam<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.env
+    }
+}
+
+impl<'a> EnvParam<'a> {
+    pub fn get_class(&self) -> &JClass<'a> {
+        &self.class
+    }
+}
+
+fn execute_and_catch<'env, F, Ret>(mut env: EnvParam<'env>, inner: F) -> Ret
+where
+    F: FnOnce(&mut EnvParam<'env>) -> Result<Ret>,
+    Ret: Default + 'env,
+{
+    match rw_catch_unwind(std::panic::AssertUnwindSafe(|| inner(&mut env))) {
+        Ok(Ok(ret)) => ret,
+        Ok(Err(e)) => {
+            match e {
+                BindingError::Jni {
+                    error: jni::errors::Error::JavaException,
+                    backtrace,
+                } => {
+                    tracing::error!("get JavaException thrown from: {:?}", backtrace);
+                    // the exception is already thrown. No need to throw again
+                }
+                _ => {
+                    env.throw(format!("get error while processing: {:?}", e))
+                        .expect("should be able to throw");
+                }
+            }
+            Ret::default()
+        }
+        Err(e) => {
+            env.throw(format!("panic while processing: {:?}", e))
+                .expect("should be able to throw");
+            Ret::default()
+        }
+    }
+}
+
+pub enum JavaBindingRowInner {
+    Keyed(KeyedRow),
+    StreamChunk(StreamChunkRow),
+}
+#[derive(Default)]
+pub struct JavaClassMethodCache {
+    big_decimal_ctor: OnceLock<(GlobalRef, JMethodID)>,
+    timestamp_ctor: OnceLock<(GlobalRef, JMethodID)>,
+
+    date_ctor: OnceLock<(GlobalRef, JStaticMethodID)>,
+    time_ctor: OnceLock<(GlobalRef, JStaticMethodID)>,
+}
+
+pub struct JavaBindingRow {
+    inner: JavaBindingRowInner,
+    class_cache: Arc<JavaClassMethodCache>,
+}
+
+impl JavaBindingRow {
+    fn with_stream_chunk(
+        underlying: StreamChunkRow,
+        class_cache: Arc<JavaClassMethodCache>,
+    ) -> Self {
+        Self {
+            inner: JavaBindingRowInner::StreamChunk(underlying),
+            class_cache,
+        }
+    }
+
+    fn with_keyed(underlying: KeyedRow, class_cache: Arc<JavaClassMethodCache>) -> Self {
+        Self {
+            inner: JavaBindingRowInner::Keyed(underlying),
+            class_cache,
+        }
+    }
+
+    fn as_keyed(&self) -> &KeyedRow {
+        match &self.inner {
+            JavaBindingRowInner::Keyed(r) => r,
+            _ => unreachable!("can only call as_keyed for KeyedRow"),
+        }
+    }
+
+    fn as_stream_chunk(&self) -> &StreamChunkRow {
+        match &self.inner {
+            JavaBindingRowInner::StreamChunk(r) => r,
+            _ => unreachable!("can only call as_stream_chunk for StreamChunkRow"),
+        }
+    }
+}
+
+impl Deref for JavaBindingRow {
+    type Target = OwnedRow;
+
+    fn deref(&self) -> &Self::Target {
+        match &self.inner {
+            JavaBindingRowInner::Keyed(r) => r.row(),
+            JavaBindingRowInner::StreamChunk(r) => r.row(),
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_vnodeCount(
+    _env: EnvParam<'_>,
+) -> jint {
+    VirtualNode::COUNT as jint
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_hummockIteratorNew<'a>(
+    env: EnvParam<'a>,
+    read_plan: JByteArray<'a>,
+) -> Pointer<'static, HummockJavaBindingIterator> {
+    execute_and_catch(env, move |env| {
+        let read_plan = Message::decode(to_guarded_slice(&read_plan, env)?.deref())?;
+        let iter = RUNTIME.block_on(HummockJavaBindingIterator::new(read_plan))?;
+        Ok(iter.into())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_hummockIteratorNext<'a>(
+    env: EnvParam<'a>,
+    mut pointer: Pointer<'a, HummockJavaBindingIterator>,
+) -> Pointer<'static, JavaBindingRow> {
+    execute_and_catch(env, move |_env| {
+        let iter = pointer.as_mut();
+        match RUNTIME.block_on(iter.next())? {
+            None => Ok(Pointer::null()),
+            Some(row) => Ok(JavaBindingRow::with_keyed(row, iter.class_cache.clone()).into()),
+        }
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_hummockIteratorClose(
+    _env: EnvParam<'_>,
+    pointer: Pointer<'_, HummockJavaBindingIterator>,
+) {
+    pointer.drop();
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorNew<'a>(
+    env: EnvParam<'a>,
+    stream_chunk_payload: JByteArray<'a>,
+) -> Pointer<'static, StreamChunkIterator> {
+    execute_and_catch(env, move |env| {
+        let prost_stream_chumk =
+            Message::decode(to_guarded_slice(&stream_chunk_payload, env)?.deref())?;
+        let iter = StreamChunkIterator::new(StreamChunk::from_protobuf(&prost_stream_chumk)?);
+        Ok(iter.into())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorFromPretty<
+    'a,
+>(
+    env: EnvParam<'a>,
+    str: JString<'a>,
+) -> Pointer<'static, StreamChunkIterator> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let iter = StreamChunkIterator::new(StreamChunk::from_pretty(
+            env.get_string(&str)
+                .expect("cannot get java string")
+                .to_str()
+                .unwrap(),
+        ));
+        Ok(iter.into())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorNext<'a>(
+    env: EnvParam<'a>,
+    mut pointer: Pointer<'a, StreamChunkIterator>,
+) -> Pointer<'static, JavaBindingRow> {
+    execute_and_catch(env, move |_env| {
+        let iter = pointer.as_mut();
+        match iter.next() {
+            None => Ok(Pointer::null()),
+            Some(row) => {
+                Ok(JavaBindingRow::with_stream_chunk(row, iter.class_cache.clone()).into())
+            }
+        }
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_streamChunkIteratorClose(
+    _env: EnvParam<'_>,
+    pointer: Pointer<'_, StreamChunkIterator>,
+) {
+    pointer.drop();
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetKey<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+) -> JByteArray<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        Ok(env.byte_array_from_slice(pointer.as_ref().as_keyed().key())?)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetOp<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+) -> jint {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer.as_ref().as_stream_chunk().op() as jint)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowIsNull<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jboolean {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer.as_ref().datum_at(idx as usize).is_none() as jboolean)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetInt16Value<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jshort {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_int16())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetInt32Value<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jint {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_int32())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetInt64Value<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jlong {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_int64())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetFloatValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jfloat {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_float32()
+            .into())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetDoubleValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jdouble {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_float64()
+            .into())
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetBooleanValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> jboolean {
+    execute_and_catch(env, move |_env| {
+        Ok(pointer.as_ref().datum_at(idx as usize).unwrap().into_bool() as jboolean)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetStringValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JString<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'a>| {
+        Ok(env.new_string(pointer.as_ref().datum_at(idx as usize).unwrap().into_utf8())?)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetIntervalValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JString<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'a>| {
+        let interval = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_interval()
+            .as_iso_8601();
+        Ok(env.new_string(interval)?)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetJsonbValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JString<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let jsonb = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_jsonb()
+            .to_string();
+        Ok(env.new_string(jsonb)?)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetTimestampValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JObject<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let scalar_value = pointer.as_ref().datum_at(idx as usize).unwrap();
+        let millis = match scalar_value {
+            // supports sinking rw timestamptz to mysql timestamp
+            ScalarRefImpl::Timestamptz(tz) => tz.timestamp_millis(),
+            ScalarRefImpl::Timestamp(ts) => ts.0.timestamp_millis(),
+            _ => panic!("expect timestamp or timestamptz"),
+        };
+        let (ts_class_ref, constructor) = pointer
+            .as_ref()
+            .class_cache
+            .timestamp_ctor
+            .get_or_try_init(|| {
+                let cls = env.find_class("java/sql/Timestamp")?;
+                let init_method = env.get_method_id(&cls, "<init>", "(J)V")?;
+                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
+            })?;
+        unsafe {
+            let ts_class = <&JClass<'_>>::from(ts_class_ref.as_obj());
+            let date_obj =
+                env.new_object_unchecked(ts_class, *constructor, &[jvalue { j: millis }])?;
+            Ok(date_obj)
+        }
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetDecimalValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JObject<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let value = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_decimal()
+            .to_string();
+        let string_value = env.new_string(value)?;
+        let (decimal_class_ref, constructor) = pointer
+            .as_ref()
+            .class_cache
+            .big_decimal_ctor
+            .get_or_try_init(|| {
+                let cls = env.find_class("java/math/BigDecimal")?;
+                let init_method = env.get_method_id(&cls, "<init>", "(Ljava/lang/String;)V")?;
+                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
+            })?;
+        unsafe {
+            let decimal_class = <&JClass<'_>>::from(decimal_class_ref.as_obj());
+            let date_obj = env.new_object_unchecked(
+                decimal_class,
+                *constructor,
+                &[jvalue {
+                    l: string_value.into_raw(),
+                }],
+            )?;
+            Ok(date_obj)
+        }
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetDateValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JObject<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let value = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_date()
+            .0
+            .to_string();
+
+        let string_value = env.new_string(value)?;
+        let (class_ref, constructor) =
+            pointer.as_ref().class_cache.date_ctor.get_or_try_init(|| {
+                let cls = env.find_class("java/sql/Date")?;
+                let init_method = env.get_static_method_id(
+                    &cls,
+                    "valueOf",
+                    "(Ljava/lang/String;)Ljava/sql/Date;",
+                )?;
+                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
+            })?;
+        unsafe {
+            let JValueOwned::Object(date_obj) = env.call_static_method_unchecked(
+                <&JClass<'_>>::from(class_ref.as_obj()),
+                *constructor,
+                ReturnType::Object,
+                &[jvalue {
+                    l: string_value.into_raw(),
+                }],
+            )?
+            else {
+                return Err(BindingError::from(jni::errors::Error::MethodNotFound {
+                    name: "valueOf".to_string(),
+                    sig: "(Ljava/lang/String;)Ljava/sql/Date;".into(),
+                }));
+            };
+            Ok(date_obj)
+        }
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetTimeValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JObject<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let value = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_time()
+            .0
+            .to_string();
+
+        let string_value = env.new_string(value)?;
+        let (class_ref, constructor) =
+            pointer.as_ref().class_cache.time_ctor.get_or_try_init(|| {
+                let cls = env.find_class("java/sql/Time")?;
+                let init_method = env.get_static_method_id(
+                    &cls,
+                    "valueOf",
+                    "(Ljava/lang/String;)Ljava/sql/Time;",
+                )?;
+                Ok::<_, jni::errors::Error>((env.new_global_ref(cls)?, init_method))
+            })?;
+        unsafe {
+            let class = <&JClass<'_>>::from(class_ref.as_obj());
+            match env.call_static_method_unchecked(
+                class,
+                *constructor,
+                ReturnType::Object,
+                &[jvalue {
+                    l: string_value.into_raw(),
+                }],
+            )? {
+                JValueGen::Object(obj) => Ok(obj),
+                _ => Err(BindingError::from(jni::errors::Error::MethodNotFound {
+                    name: "valueOf".to_string(),
+                    sig: "(Ljava/lang/String;)Ljava/sql/Time;".into(),
+                })),
+            }
+        }
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetByteaValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+) -> JByteArray<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let bytes = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_bytea();
+        Ok(env.byte_array_from_slice(bytes)?)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowGetArrayValue<'a>(
+    env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+    idx: jint,
+    class: JClass<'a>,
+) -> JObject<'a> {
+    execute_and_catch(env, move |env: &mut EnvParam<'_>| {
+        let elems = pointer
+            .as_ref()
+            .datum_at(idx as usize)
+            .unwrap()
+            .into_list()
+            .iter();
+
+        // convert the Rust elements to a Java object array (Object[])
+        let jarray = env.new_object_array(elems.len() as jsize, &class, JObject::null())?;
+
+        for (i, ele) in elems.enumerate() {
+            let index = i as jsize;
+            match ele {
+                None => env.set_object_array_element(&jarray, i as jsize, JObject::null())?,
+                Some(val) => match val {
+                    ScalarRefImpl::Int16(v) => {
+                        let obj = env.call_static_method(
+                            &class,
+                            "valueOf",
+                            "(S)Ljava.lang.Short;",
+                            &[JValue::from(v as jshort)],
+                        )?;
+                        if let JValueOwned::Object(o) = obj {
+                            env.set_object_array_element(&jarray, index, &o)?
+                        }
+                    }
+                    ScalarRefImpl::Int32(v) => {
+                        let obj = env.call_static_method(
+                            &class,
+                            "valueOf",
+                            "(I)Ljava.lang.Integer;",
+                            &[JValue::from(v as jint)],
+                        )?;
+                        if let JValueOwned::Object(o) = obj {
+                            env.set_object_array_element(&jarray, index, &o)?
+                        }
+                    }
+                    ScalarRefImpl::Int64(v) => {
+                        let obj = env.call_static_method(
+                            &class,
+                            "valueOf",
+                            "(J)Ljava.lang.Long;",
+                            &[JValue::from(v as jlong)],
+                        )?;
+                        if let JValueOwned::Object(o) = obj {
+                            env.set_object_array_element(&jarray, index, &o)?
+                        }
+                    }
+                    ScalarRefImpl::Float32(v) => {
+                        let obj = env.call_static_method(
+                            &class,
+                            "valueOf",
+                            "(F)Ljava/lang/Float;",
+                            &[JValue::from(v.into_inner() as jfloat)],
+                        )?;
+                        if let JValueOwned::Object(o) = obj {
+                            env.set_object_array_element(&jarray, index, &o)?
+                        }
+                    }
+                    ScalarRefImpl::Float64(v) => {
+                        let obj = env.call_static_method(
+                            &class,
+                            "valueOf",
+                            "(D)Ljava/lang/Double;",
+                            &[JValue::from(v.into_inner() as jdouble)],
+                        )?;
+                        if let JValueOwned::Object(o) = obj {
+                            env.set_object_array_element(&jarray, index, &o)?
+                        }
+                    }
+                    ScalarRefImpl::Utf8(v) => {
+                        let obj = env.new_string(v)?;
+                        env.set_object_array_element(&jarray, index, obj)?
+                    }
+                    _ => env.set_object_array_element(&jarray, index, JObject::null())?,
+                },
+            }
+        }
+        let output = unsafe { JObject::from_raw(jarray.into_raw()) };
+        Ok(output)
+    })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_risingwave_java_binding_Binding_rowClose<'a>(
+    _env: EnvParam<'a>,
+    pointer: Pointer<'a, JavaBindingRow>,
+) {
+    pointer.drop()
+}
+
+#[cfg(test)]
+mod tests {
+    use risingwave_common::types::{DataType, Timestamptz};
+    use risingwave_expr::vector_op::cast::literal_parsing;
+
+    /// make sure that the [`ScalarRefImpl::Int64`] received by
+    /// [`Java_com_risingwave_java_binding_Binding_rowGetTimestampValue`]
+    /// is of type [`DataType::Timestamptz`] stored in microseconds
+    #[test]
+    fn test_timestamptz_to_i64() {
+        assert_eq!(
+            literal_parsing(&DataType::Timestamptz, "2023-06-01 09:45:00+08:00").unwrap(),
+            Timestamptz::from_micros(1_685_583_900_000_000).into()
+        );
+    }
+}
diff --git a/src/java_binding/src/stream_chunk_iterator.rs b/src/jni_core/src/stream_chunk_iterator.rs
similarity index 100%
rename from src/java_binding/src/stream_chunk_iterator.rs
rename to src/jni_core/src/stream_chunk_iterator.rs
diff --git a/src/meta/src/backup_restore/backup_manager.rs b/src/meta/src/backup_restore/backup_manager.rs
index e990741221ef8..1eea48307cb33 100644
--- a/src/meta/src/backup_restore/backup_manager.rs
+++ b/src/meta/src/backup_restore/backup_manager.rs
@@ -33,7 +33,6 @@ use crate::backup_restore::metrics::BackupManagerMetrics;
 use crate::hummock::{HummockManagerRef, HummockVersionSafePoint};
 use crate::manager::{IdCategory, LocalNotification, MetaSrvEnv};
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
 pub enum BackupJobResult {
@@ -59,14 +58,14 @@ impl BackupJobHandle {
     }
 }
 
-pub type BackupManagerRef<S> = Arc<BackupManager<S>>;
+pub type BackupManagerRef = Arc<BackupManager>;
 /// (url, dir)
 type StoreConfig = (String, String);
 
 /// `BackupManager` manages lifecycle of all existent backups and the running backup job.
-pub struct BackupManager<S: MetaStore> {
-    env: MetaSrvEnv<S>,
-    hummock_manager: HummockManagerRef<S>,
+pub struct BackupManager {
+    env: MetaSrvEnv,
+    hummock_manager: HummockManagerRef,
     backup_store: ArcSwap<(BoxedMetaSnapshotStorage, StoreConfig)>,
     /// Tracks the running backup job. Concurrent jobs is not supported.
     running_backup_job: tokio::sync::Mutex<Option<BackupJobHandle>>,
@@ -74,10 +73,10 @@ pub struct BackupManager<S: MetaStore> {
     meta_metrics: Arc<MetaMetrics>,
 }
 
-impl<S: MetaStore> BackupManager<S> {
+impl BackupManager {
     pub async fn new(
-        env: MetaSrvEnv<S>,
-        hummock_manager: HummockManagerRef<S>,
+        env: MetaSrvEnv,
+        hummock_manager: HummockManagerRef,
         metrics: Arc<MetaMetrics>,
         store_url: &str,
         store_dir: &str,
@@ -139,8 +138,8 @@ impl<S: MetaStore> BackupManager<S> {
     }
 
     fn with_store(
-        env: MetaSrvEnv<S>,
-        hummock_manager: HummockManagerRef<S>,
+        env: MetaSrvEnv,
+        hummock_manager: HummockManagerRef,
         meta_metrics: Arc<MetaMetrics>,
         backup_store: (BoxedMetaSnapshotStorage, StoreConfig),
     ) -> Self {
@@ -167,7 +166,7 @@ impl<S: MetaStore> BackupManager<S> {
     }
 
     #[cfg(test)]
-    pub fn for_test(env: MetaSrvEnv<S>, hummock_manager: HummockManagerRef<S>) -> Self {
+    pub fn for_test(env: MetaSrvEnv, hummock_manager: HummockManagerRef) -> Self {
         Self::with_store(
             env,
             hummock_manager,
@@ -326,12 +325,12 @@ impl<S: MetaStore> BackupManager<S> {
 }
 
 /// `BackupWorker` creates a database snapshot.
-struct BackupWorker<S: MetaStore> {
-    backup_manager: BackupManagerRef<S>,
+struct BackupWorker {
+    backup_manager: BackupManagerRef,
 }
 
-impl<S: MetaStore> BackupWorker<S> {
-    fn new(backup_manager: BackupManagerRef<S>) -> Self {
+impl BackupWorker {
+    fn new(backup_manager: BackupManagerRef) -> Self {
         Self { backup_manager }
     }
 
diff --git a/src/meta/src/backup_restore/meta_snapshot_builder.rs b/src/meta/src/backup_restore/meta_snapshot_builder.rs
index 6060da1ec1c51..0df9966607b93 100644
--- a/src/meta/src/backup_restore/meta_snapshot_builder.rs
+++ b/src/meta/src/backup_restore/meta_snapshot_builder.rs
@@ -14,7 +14,6 @@
 
 use std::collections::HashMap;
 use std::future::Future;
-use std::sync::Arc;
 
 use anyhow::anyhow;
 use risingwave_backup::error::{BackupError, BackupResult};
@@ -36,11 +35,11 @@ const VERSION: u32 = 1;
 
 pub struct MetaSnapshotBuilder<S> {
     snapshot: MetaSnapshot,
-    meta_store: Arc<S>,
+    meta_store: S,
 }
 
 impl<S: MetaStore> MetaSnapshotBuilder<S> {
-    pub fn new(meta_store: Arc<S>) -> Self {
+    pub fn new(meta_store: S) -> Self {
         Self {
             snapshot: MetaSnapshot::default(),
             meta_store,
@@ -161,8 +160,6 @@ impl<S: MetaStore> MetaSnapshotBuilder<S> {
 
 #[cfg(test)]
 mod tests {
-    use std::ops::Deref;
-    use std::sync::Arc;
 
     use assert_matches::assert_matches;
     use itertools::Itertools;
@@ -179,7 +176,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_snapshot_builder() {
-        let meta_store = Arc::new(MemStore::new());
+        let meta_store = MemStore::new();
 
         let mut builder = MetaSnapshotBuilder::new(meta_store.clone());
         let hummock_version = HummockVersion {
@@ -190,7 +187,7 @@ mod tests {
             let v_ = v.clone();
             async move { v_ }
         };
-        hummock_version.insert(meta_store.deref()).await.unwrap();
+        hummock_version.insert(&meta_store).await.unwrap();
         let err = builder
             .build(1, get_ckpt_builder(&hummock_version))
             .await
@@ -205,10 +202,7 @@ mod tests {
             hummock_version_id: hummock_version.id,
             ..Default::default()
         };
-        hummock_version_stats
-            .insert(meta_store.deref())
-            .await
-            .unwrap();
+        hummock_version_stats.insert(&meta_store).await.unwrap();
         let err = builder
             .build(1, get_ckpt_builder(&hummock_version))
             .await
@@ -216,10 +210,7 @@ mod tests {
         let err = assert_matches!(err, BackupError::Other(e) => e);
         assert_eq!("system params not found in meta store", err.to_error_str());
 
-        system_params_for_test()
-            .insert(meta_store.deref())
-            .await
-            .unwrap();
+        system_params_for_test().insert(&meta_store).await.unwrap();
 
         let err = builder
             .build(1, get_ckpt_builder(&hummock_version))
@@ -229,7 +220,7 @@ mod tests {
         assert_eq!("cluster id not found in meta store", err.to_error_str());
 
         ClusterId::new()
-            .put_at_meta_store(meta_store.deref())
+            .put_at_meta_store(&meta_store)
             .await
             .unwrap();
 
diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs
index d99ee11f482c9..bac77639ba1c5 100644
--- a/src/meta/src/backup_restore/restore.rs
+++ b/src/meta/src/backup_restore/restore.rs
@@ -291,6 +291,7 @@ mod tests {
     use clap::Parser;
     use itertools::Itertools;
     use risingwave_backup::meta_snapshot::{ClusterMetadata, MetaSnapshot};
+    use risingwave_common::config::SystemConfig;
     use risingwave_pb::hummock::HummockVersion;
     use risingwave_pb::meta::SystemParams;
 
@@ -318,18 +319,9 @@ mod tests {
 
     fn get_system_params() -> SystemParams {
         SystemParams {
-            barrier_interval_ms: Some(101),
-            checkpoint_frequency: Some(102),
-            sstable_size_mb: Some(103),
-            block_size_kb: Some(104),
-            bloom_false_positive: Some(0.1),
             state_store: Some("state_store".to_string()),
             data_directory: Some("data_directory".to_string()),
-            backup_storage_url: Some("backup_storage_url".to_string()),
-            backup_storage_directory: Some("backup_storage_directory".to_string()),
-            telemetry_enabled: Some(false),
-            parallel_compact_size_mb: Some(255),
-            max_concurrent_creating_streaming_jobs: Some(1),
+            ..SystemConfig::default().into_init_system_params()
         }
     }
 
diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs
index eb8255b28db88..d0deac65b3207 100644
--- a/src/meta/src/barrier/command.rs
+++ b/src/meta/src/barrier/command.rs
@@ -21,6 +21,7 @@ use risingwave_common::catalog::TableId;
 use risingwave_common::hash::ActorMapping;
 use risingwave_connector::source::SplitImpl;
 use risingwave_hummock_sdk::HummockEpoch;
+use risingwave_pb::meta::PausedReason;
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
 use risingwave_pb::stream_plan::barrier::{BarrierKind, Mutation};
 use risingwave_pb::stream_plan::update_mutation::*;
@@ -37,7 +38,6 @@ use super::trace::TracedEpoch;
 use crate::barrier::CommandChanges;
 use crate::manager::{FragmentManagerRef, WorkerId};
 use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments};
-use crate::storage::MetaStore;
 use crate::stream::{build_actor_connector_splits, SourceManagerRef, SplitAssignment};
 use crate::MetaResult;
 
@@ -79,6 +79,15 @@ pub enum Command {
     /// After the barrier is collected, it does nothing.
     Plain(Option<Mutation>),
 
+    /// `Pause` command generates a `Pause` barrier with the provided [`PausedReason`] **only if**
+    /// the cluster is not already paused. Otherwise, a barrier with no mutation will be generated.
+    Pause(PausedReason),
+
+    /// `Resume` command generates a `Resume` barrier with the provided [`PausedReason`] **only
+    /// if** the cluster is paused with the same reason. Otherwise, a barrier with no mutation
+    /// will be generated.
+    Resume(PausedReason),
+
     /// `DropStreamingJobs` command generates a `Stop` barrier by the given
     /// [`HashSet<TableId>`]. The catalog has ensured that these streaming jobs are safe to be
     /// dropped by reference counts before.
@@ -142,18 +151,20 @@ impl Command {
         Self::Plain(None)
     }
 
-    pub fn pause() -> Self {
-        Self::Plain(Some(Mutation::Pause(PauseMutation {})))
+    pub fn pause(reason: PausedReason) -> Self {
+        Self::Pause(reason)
     }
 
-    pub fn resume() -> Self {
-        Self::Plain(Some(Mutation::Resume(ResumeMutation {})))
+    pub fn resume(reason: PausedReason) -> Self {
+        Self::Resume(reason)
     }
 
     /// Changes to the actors to be sent or collected after this command is committed.
     pub fn changes(&self) -> CommandChanges {
         match self {
             Command::Plain(_) => CommandChanges::None,
+            Command::Pause(_) => CommandChanges::None,
+            Command::Resume(_) => CommandChanges::None,
             Command::CreateStreamingJob {
                 table_fragments, ..
             } => CommandChanges::CreateTable(table_fragments.table_id()),
@@ -189,22 +200,22 @@ impl Command {
     /// injection. return true.
     pub fn should_pause_inject_barrier(&self) -> bool {
         // Note: the meaning for `Pause` is not pausing the periodic barrier injection, but for
-        // pausing the sources on compute nodes. However, `Pause` is used for configuration change
-        // like scaling and migration, which must pause the concurrent checkpoint to ensure the
+        // pausing the sources on compute nodes. However, when `Pause` is used for configuration
+        // change like scaling and migration, it must pause the concurrent checkpoint to ensure the
         // previous checkpoint has been done.
-        matches!(self, Self::Plain(Some(Mutation::Pause(_))))
+        matches!(self, Self::Pause(PausedReason::ConfigChange))
     }
 
     pub fn need_checkpoint(&self) -> bool {
         // todo! Reviewing the flow of different command to reduce the amount of checkpoint
-        !matches!(self, Command::Plain(None | Some(Mutation::Resume(_))))
+        !matches!(self, Command::Plain(None) | Command::Resume(_))
     }
 }
 
 /// [`CommandContext`] is used for generating barrier and doing post stuffs according to the given
 /// [`Command`].
-pub struct CommandContext<S: MetaStore> {
-    fragment_manager: FragmentManagerRef<S>,
+pub struct CommandContext {
+    fragment_manager: FragmentManagerRef,
 
     client_pool: StreamClientPoolRef,
 
@@ -215,11 +226,13 @@ pub struct CommandContext<S: MetaStore> {
     pub prev_epoch: TracedEpoch,
     pub curr_epoch: TracedEpoch,
 
+    pub current_paused_reason: Option<PausedReason>,
+
     pub command: Command,
 
     pub kind: BarrierKind,
 
-    source_manager: SourceManagerRef<S>,
+    source_manager: SourceManagerRef,
 
     /// The tracing span of this command.
     ///
@@ -229,17 +242,18 @@ pub struct CommandContext<S: MetaStore> {
     pub span: tracing::Span,
 }
 
-impl<S: MetaStore> CommandContext<S> {
+impl CommandContext {
     #[allow(clippy::too_many_arguments)]
     pub(super) fn new(
-        fragment_manager: FragmentManagerRef<S>,
+        fragment_manager: FragmentManagerRef,
         client_pool: StreamClientPoolRef,
         info: BarrierActorInfo,
         prev_epoch: TracedEpoch,
         curr_epoch: TracedEpoch,
+        current_paused_reason: Option<PausedReason>,
         command: Command,
         kind: BarrierKind,
-        source_manager: SourceManagerRef<S>,
+        source_manager: SourceManagerRef,
         span: tracing::Span,
     ) -> Self {
         Self {
@@ -248,6 +262,7 @@ impl<S: MetaStore> CommandContext<S> {
             info: Arc::new(info),
             prev_epoch,
             curr_epoch,
+            current_paused_reason,
             command,
             kind,
             source_manager,
@@ -256,15 +271,30 @@ impl<S: MetaStore> CommandContext<S> {
     }
 }
 
-impl<S> CommandContext<S>
-where
-    S: MetaStore,
-{
+impl CommandContext {
     /// Generate a mutation for the given command.
     pub async fn to_mutation(&self) -> MetaResult<Option<Mutation>> {
         let mutation = match &self.command {
             Command::Plain(mutation) => mutation.clone(),
 
+            Command::Pause(_) => {
+                // Only pause when the cluster is not already paused.
+                if self.current_paused_reason.is_none() {
+                    Some(Mutation::Pause(PauseMutation {}))
+                } else {
+                    None
+                }
+            }
+
+            Command::Resume(reason) => {
+                // Only resume when the cluster is paused with the same reason.
+                if self.current_paused_reason == Some(*reason) {
+                    Some(Mutation::Resume(ResumeMutation {}))
+                } else {
+                    None
+                }
+            }
+
             Command::SourceSplitAssignment(change) => {
                 let mut diff = HashMap::new();
 
@@ -308,6 +338,8 @@ where
                     actor_dispatchers,
                     added_actors,
                     actor_splits,
+                    // If the cluster is already paused, the new actors should be paused too.
+                    pause: self.current_paused_reason.is_some(),
                 }))
             }
 
@@ -478,6 +510,31 @@ where
         Ok(mutation)
     }
 
+    /// Returns the paused reason after executing the current command.
+    pub fn next_paused_reason(&self) -> Option<PausedReason> {
+        match &self.command {
+            Command::Pause(reason) => {
+                // Only pause when the cluster is not already paused.
+                if self.current_paused_reason.is_none() {
+                    Some(*reason)
+                } else {
+                    self.current_paused_reason
+                }
+            }
+
+            Command::Resume(reason) => {
+                // Only resume when the cluster is paused with the same reason.
+                if self.current_paused_reason == Some(*reason) {
+                    None
+                } else {
+                    self.current_paused_reason
+                }
+            }
+
+            _ => self.current_paused_reason,
+        }
+    }
+
     /// For `CreateStreamingJob`, returns the actors of the `Chain` nodes. For other commands,
     /// returns an empty set.
     pub fn actors_to_track(&self) -> HashSet<ActorId> {
@@ -562,18 +619,19 @@ where
     /// the given command.
     pub async fn post_collect(&self) -> MetaResult<()> {
         match &self.command {
-            #[allow(clippy::single_match)]
-            Command::Plain(mutation) => match mutation {
-                // After the `Pause` barrier is collected and committed, we must ensure that the
-                // storage version with this epoch is synced to all compute nodes before the
-                // execution of the next command of `Update`, as some newly created operators may
-                // immediately initialize their states on that barrier.
-                Some(Mutation::Pause(..)) => {
+            Command::Plain(_) => {}
+
+            Command::Pause(reason) => {
+                if let PausedReason::ConfigChange = reason {
+                    // After the `Pause` barrier is collected and committed, we must ensure that the
+                    // storage version with this epoch is synced to all compute nodes before the
+                    // execution of the next command of `Update`, as some newly created operators
+                    // may immediately initialize their states on that barrier.
                     self.wait_epoch_commit(self.prev_epoch.value().0).await?;
                 }
+            }
 
-                _ => {}
-            },
+            Command::Resume(_) => {}
 
             Command::SourceSplitAssignment(split_assignment) => {
                 self.fragment_manager
diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs
index a74cfc55860d5..cd3ee0360009f 100644
--- a/src/meta/src/barrier/mod.rs
+++ b/src/meta/src/barrier/mod.rs
@@ -25,11 +25,13 @@ use itertools::Itertools;
 use prometheus::HistogramTimer;
 use risingwave_common::bail;
 use risingwave_common::catalog::TableId;
+use risingwave_common::system_param::PAUSE_ON_NEXT_BOOTSTRAP_KEY;
 use risingwave_common::util::tracing::TracingContext;
 use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId};
 use risingwave_pb::ddl_service::DdlProgress;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::table_fragments::actor_status::ActorState;
+use risingwave_pb::meta::PausedReason;
 use risingwave_pb::stream_plan::barrier::BarrierKind;
 use risingwave_pb::stream_plan::Barrier;
 use risingwave_pb::stream_service::{
@@ -47,6 +49,7 @@ use self::command::CommandContext;
 use self::info::BarrierActorInfo;
 use self::notifier::Notifier;
 use self::progress::TrackingCommand;
+use crate::barrier::notifier::BarrierInfo;
 use crate::barrier::progress::CreateMviewProgressTracker;
 use crate::barrier::BarrierEpochState::{Completed, InFlight};
 use crate::hummock::HummockManagerRef;
@@ -57,7 +60,6 @@ use crate::manager::{
 };
 use crate::model::{ActorId, BarrierManagerState};
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::meta_store::MetaStore;
 use crate::stream::SourceManagerRef;
 use crate::{MetaError, MetaResult};
 
@@ -123,7 +125,7 @@ pub enum CommandChanges {
 /// accepting [`Command`] that carries info to build `Mutation`. To keep the consistency between
 /// barrier manager and meta store, some actions like "drop materialized view" or "create mv on mv"
 /// must be done in barrier manager transactional using [`Command`].
-pub struct GlobalBarrierManager<S: MetaStore> {
+pub struct GlobalBarrierManager {
     /// Enable recovery or not when failover.
     enable_recovery: bool,
 
@@ -135,29 +137,29 @@ pub struct GlobalBarrierManager<S: MetaStore> {
     /// The max barrier nums in flight
     in_flight_barrier_nums: usize,
 
-    cluster_manager: ClusterManagerRef<S>,
+    cluster_manager: ClusterManagerRef,
 
-    pub(crate) catalog_manager: CatalogManagerRef<S>,
+    pub(crate) catalog_manager: CatalogManagerRef,
 
-    fragment_manager: FragmentManagerRef<S>,
+    fragment_manager: FragmentManagerRef,
 
-    hummock_manager: HummockManagerRef<S>,
+    hummock_manager: HummockManagerRef,
 
-    source_manager: SourceManagerRef<S>,
+    source_manager: SourceManagerRef,
 
     sink_manager: SinkCoordinatorManager,
 
     metrics: Arc<MetaMetrics>,
 
-    pub(crate) env: MetaSrvEnv<S>,
+    pub(crate) env: MetaSrvEnv,
 
-    tracker: Mutex<CreateMviewProgressTracker<S>>,
+    tracker: Mutex<CreateMviewProgressTracker>,
 }
 
 /// Controls the concurrent execution of commands.
-struct CheckpointControl<S: MetaStore> {
+struct CheckpointControl {
     /// Save the state and message of barrier in order.
-    command_ctx_queue: VecDeque<EpochNode<S>>,
+    command_ctx_queue: VecDeque<EpochNode>,
 
     // Below for uncommitted changes for the inflight barriers.
     /// In addition to the actors with status `Running`. The barrier needs to send or collect the
@@ -175,13 +177,10 @@ struct CheckpointControl<S: MetaStore> {
     metrics: Arc<MetaMetrics>,
 
     /// Get notified when we finished Create MV and collect a barrier(checkpoint = true)
-    finished_commands: Vec<TrackingCommand<S>>,
+    finished_commands: Vec<TrackingCommand>,
 }
 
-impl<S> CheckpointControl<S>
-where
-    S: MetaStore,
-{
+impl CheckpointControl {
     fn new(metrics: Arc<MetaMetrics>) -> Self {
         Self {
             command_ctx_queue: Default::default(),
@@ -195,7 +194,7 @@ where
     }
 
     /// Stash a command to finish later.
-    fn stash_command_to_finish(&mut self, finished_command: TrackingCommand<S>) {
+    fn stash_command_to_finish(&mut self, finished_command: TrackingCommand) {
         self.finished_commands.push(finished_command);
     }
 
@@ -219,7 +218,7 @@ where
         Ok(!self.finished_commands.is_empty())
     }
 
-    fn cancel_command(&mut self, cancelled_command: TrackingCommand<S>) {
+    fn cancel_command(&mut self, cancelled_command: TrackingCommand) {
         if let Some(index) = self.command_ctx_queue.iter().position(|x| {
             x.command_ctx.prev_epoch.value() == cancelled_command.context.prev_epoch.value()
         }) {
@@ -322,7 +321,7 @@ where
     }
 
     /// Enqueue a barrier command, and init its state to `InFlight`.
-    fn enqueue_command(&mut self, command_ctx: Arc<CommandContext<S>>, notifiers: Vec<Notifier>) {
+    fn enqueue_command(&mut self, command_ctx: Arc<CommandContext>, notifiers: Vec<Notifier>) {
         let timer = self.metrics.barrier_latency.start_timer();
 
         self.command_ctx_queue.push_back(EpochNode {
@@ -341,7 +340,7 @@ where
         &mut self,
         prev_epoch: u64,
         result: Vec<BarrierCompleteResponse>,
-    ) -> Vec<EpochNode<S>> {
+    ) -> Vec<EpochNode> {
         // change state to complete, and wait for nodes with the smaller epoch to commit
         let wait_commit_timer = self.metrics.barrier_wait_commit_latency.start_timer();
         if let Some(node) = self
@@ -367,7 +366,7 @@ where
     }
 
     /// Remove all nodes from queue and return them.
-    fn barrier_failed(&mut self) -> Vec<EpochNode<S>> {
+    fn barrier_failed(&mut self) -> Vec<EpochNode> {
         let complete_nodes = self.command_ctx_queue.drain(..).collect_vec();
         complete_nodes
             .iter()
@@ -452,7 +451,7 @@ where
 }
 
 /// The state and message of this barrier, a node for concurrent checkpoint.
-pub struct EpochNode<S: MetaStore> {
+pub struct EpochNode {
     /// Timer for recording barrier latency, taken after `complete_barriers`.
     timer: Option<HistogramTimer>,
     /// The timer of `barrier_wait_commit_latency`
@@ -461,7 +460,7 @@ pub struct EpochNode<S: MetaStore> {
     /// Whether this barrier is in-flight or completed.
     state: BarrierEpochState,
     /// Context of this command to generate barrier and do some post jobs.
-    command_ctx: Arc<CommandContext<S>>,
+    command_ctx: Arc<CommandContext>,
     /// Notifiers of this barrier.
     notifiers: Vec<Notifier>,
 }
@@ -482,20 +481,17 @@ struct BarrierCompletion {
     result: MetaResult<Vec<BarrierCompleteResponse>>,
 }
 
-impl<S> GlobalBarrierManager<S>
-where
-    S: MetaStore,
-{
+impl GlobalBarrierManager {
     /// Create a new [`crate::barrier::GlobalBarrierManager`].
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         scheduled_barriers: schedule::ScheduledBarriers,
-        env: MetaSrvEnv<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        catalog_manager: CatalogManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
-        hummock_manager: HummockManagerRef<S>,
-        source_manager: SourceManagerRef<S>,
+        env: MetaSrvEnv,
+        cluster_manager: ClusterManagerRef,
+        catalog_manager: CatalogManagerRef,
+        fragment_manager: FragmentManagerRef,
+        hummock_manager: HummockManagerRef,
+        source_manager: SourceManagerRef,
         sink_manager: SinkCoordinatorManager,
         metrics: Arc<MetaMetrics>,
     ) -> Self {
@@ -520,7 +516,7 @@ where
         }
     }
 
-    pub fn start(barrier_manager: BarrierManagerRef<S>) -> (JoinHandle<()>, Sender<()>) {
+    pub fn start(barrier_manager: BarrierManagerRef) -> (JoinHandle<()>, Sender<()>) {
         let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
         let join_handle = tokio::spawn(async move {
             barrier_manager.run(shutdown_rx).await;
@@ -541,6 +537,23 @@ where
         *status = new_status;
     }
 
+    /// Check whether we should pause on bootstrap from the system parameter and reset it.
+    async fn take_pause_on_bootstrap(&self) -> MetaResult<bool> {
+        let pm = self.env.system_params_manager();
+        let paused = pm.get_params().await.pause_on_next_bootstrap();
+        if paused {
+            tracing::warn!(
+                "The cluster will bootstrap with all data sources paused as specified by the system parameter `{}`. \
+                 It will now be reset to `false`. \
+                 To resume the data sources, either restart the cluster again or use `risectl meta resume`.",
+                PAUSE_ON_NEXT_BOOTSTRAP_KEY
+            );
+            pm.set_param(PAUSE_ON_NEXT_BOOTSTRAP_KEY, Some("false".to_owned()))
+                .await?;
+        }
+        Ok(paused)
+    }
+
     /// Start an infinite loop to take scheduled barriers and send them.
     async fn run(&self, mut shutdown_rx: Receiver<()>) {
         // Initialize the barrier manager.
@@ -579,9 +592,13 @@ where
             // inject the first `Initial` barrier.
             self.set_status(BarrierManagerStatus::Recovering).await;
             let span = tracing::info_span!("bootstrap_recovery", prev_epoch = prev_epoch.value().0);
-            let new_epoch = self.recovery(prev_epoch).instrument(span).await;
 
-            BarrierManagerState::new(new_epoch)
+            let paused = self.take_pause_on_bootstrap().await.unwrap_or(false);
+            let paused_reason = paused.then_some(PausedReason::Manual);
+
+            self.recovery(prev_epoch, paused_reason)
+                .instrument(span)
+                .await
         };
 
         self.set_status(BarrierManagerStatus::Running).await;
@@ -650,13 +667,13 @@ where
         &self,
         barrier_complete_tx: &UnboundedSender<BarrierCompletion>,
         state: &mut BarrierManagerState,
-        checkpoint_control: &mut CheckpointControl<S>,
+        checkpoint_control: &mut CheckpointControl,
     ) {
         assert!(checkpoint_control.can_inject_barrier(self.in_flight_barrier_nums));
 
         let Scheduled {
             command,
-            notifiers,
+            mut notifiers,
             send_latency_timer,
             checkpoint,
             span,
@@ -680,27 +697,43 @@ where
             self.fragment_manager.clone(),
             self.env.stream_client_pool_ref(),
             info,
-            prev_epoch,
-            curr_epoch,
+            prev_epoch.clone(),
+            curr_epoch.clone(),
+            state.paused_reason(),
             command,
             kind,
             self.source_manager.clone(),
             span.clone(),
         ));
-        let mut notifiers = notifiers;
-        notifiers.iter_mut().for_each(Notifier::notify_to_send);
+
         send_latency_timer.observe_duration();
 
-        checkpoint_control.enqueue_command(command_ctx.clone(), notifiers);
-        self.inject_barrier(command_ctx, barrier_complete_tx)
+        self.inject_barrier(command_ctx.clone(), barrier_complete_tx)
             .instrument(span)
             .await;
+
+        // Notify about the injection.
+        let prev_paused_reason = state.paused_reason();
+        let curr_paused_reason = command_ctx.next_paused_reason();
+
+        let info = BarrierInfo {
+            prev_epoch: prev_epoch.value(),
+            curr_epoch: curr_epoch.value(),
+            prev_paused_reason,
+            curr_paused_reason,
+        };
+        notifiers.iter_mut().for_each(|n| n.notify_injected(info));
+
+        // Update the paused state after the barrier is injected.
+        state.set_paused_reason(curr_paused_reason);
+        // Record the in-flight barrier.
+        checkpoint_control.enqueue_command(command_ctx.clone(), notifiers);
     }
 
     /// Inject a barrier to all CNs and spawn a task to collect it
     async fn inject_barrier(
         &self,
-        command_context: Arc<CommandContext<S>>,
+        command_context: Arc<CommandContext>,
         barrier_complete_tx: &UnboundedSender<BarrierCompletion>,
     ) {
         let prev_epoch = command_context.prev_epoch.value().0;
@@ -727,7 +760,7 @@ where
     /// Send inject-barrier-rpc to stream service and wait for its response before returns.
     async fn inject_barrier_inner(
         &self,
-        command_context: Arc<CommandContext<S>>,
+        command_context: Arc<CommandContext>,
     ) -> MetaResult<HashMap<WorkerId, bool>> {
         fail_point!("inject_barrier_err", |_| bail!("inject_barrier_err"));
         let mutation = command_context.to_mutation().await?;
@@ -784,7 +817,7 @@ where
     async fn collect_barrier(
         node_need_collect: HashMap<WorkerId, bool>,
         client_pool_ref: StreamClientPoolRef,
-        command_context: Arc<CommandContext<S>>,
+        command_context: Arc<CommandContext>,
         barrier_complete_tx: UnboundedSender<BarrierCompletion>,
     ) {
         let prev_epoch = command_context.prev_epoch.value().0;
@@ -831,7 +864,7 @@ where
         &self,
         completion: BarrierCompletion,
         state: &mut BarrierManagerState,
-        checkpoint_control: &mut CheckpointControl<S>,
+        checkpoint_control: &mut CheckpointControl,
     ) {
         let BarrierCompletion { prev_epoch, result } = completion;
 
@@ -887,9 +920,9 @@ where
     async fn failure_recovery(
         &self,
         err: MetaError,
-        fail_nodes: impl IntoIterator<Item = EpochNode<S>>,
+        fail_nodes: impl IntoIterator<Item = EpochNode>,
         state: &mut BarrierManagerState,
-        checkpoint_control: &mut CheckpointControl<S>,
+        checkpoint_control: &mut CheckpointControl,
     ) {
         checkpoint_control.clear_changes();
 
@@ -918,9 +951,8 @@ where
                 %err,
                 prev_epoch = prev_epoch.value().0
             );
-            let new_epoch = self.recovery(prev_epoch).instrument(span).await;
 
-            *state = BarrierManagerState::new(new_epoch);
+            *state = self.recovery(prev_epoch, None).instrument(span).await;
             self.set_status(BarrierManagerStatus::Running).await;
         } else {
             panic!("failed to execute barrier: {:?}", err);
@@ -930,8 +962,8 @@ where
     /// Try to commit this node. If err, returns
     async fn complete_barrier(
         &self,
-        node: &mut EpochNode<S>,
-        checkpoint_control: &mut CheckpointControl<S>,
+        node: &mut EpochNode,
+        checkpoint_control: &mut CheckpointControl,
     ) -> MetaResult<()> {
         let prev_epoch = node.command_ctx.prev_epoch.value().0;
         match &mut node.state {
@@ -1055,7 +1087,7 @@ where
     /// will create or drop before this barrier flow through them.
     async fn resolve_actor_info(
         &self,
-        checkpoint_control: &mut CheckpointControl<S>,
+        checkpoint_control: &mut CheckpointControl,
         command: &Command,
     ) -> BarrierActorInfo {
         checkpoint_control.pre_resolve(command);
@@ -1081,7 +1113,7 @@ where
     }
 }
 
-pub type BarrierManagerRef<S> = Arc<GlobalBarrierManager<S>>;
+pub type BarrierManagerRef = Arc<GlobalBarrierManager>;
 
 fn collect_synced_ssts(
     resps: &mut [BarrierCompleteResponse],
diff --git a/src/meta/src/barrier/notifier.rs b/src/meta/src/barrier/notifier.rs
index 640886079854c..88acd9cd3dd7a 100644
--- a/src/meta/src/barrier/notifier.rs
+++ b/src/meta/src/barrier/notifier.rs
@@ -12,15 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::util::epoch::Epoch;
+use risingwave_pb::meta::PausedReason;
 use tokio::sync::oneshot;
 
 use crate::{MetaError, MetaResult};
 
+/// The barrier info sent back to the caller when a barrier is injected.
+#[derive(Debug, Clone, Copy)]
+pub struct BarrierInfo {
+    pub prev_epoch: Epoch,
+    pub curr_epoch: Epoch,
+
+    pub prev_paused_reason: Option<PausedReason>,
+    pub curr_paused_reason: Option<PausedReason>,
+}
+
 /// Used for notifying the status of a scheduled command/barrier.
 #[derive(Debug, Default)]
 pub(super) struct Notifier {
-    /// Get notified when scheduled barrier is about to send.
-    pub to_send: Option<oneshot::Sender<()>>,
+    /// Get notified when scheduled barrier is injected to compute nodes.
+    pub injected: Option<oneshot::Sender<BarrierInfo>>,
 
     /// Get notified when scheduled barrier is collected or failed.
     pub collected: Option<oneshot::Sender<MetaResult<()>>>,
@@ -30,10 +42,10 @@ pub(super) struct Notifier {
 }
 
 impl Notifier {
-    /// Notify when we are about to send a barrier.
-    pub fn notify_to_send(&mut self) {
-        if let Some(tx) = self.to_send.take() {
-            tx.send(()).ok();
+    /// Notify when we have injected a barrier to compute nodes.
+    pub fn notify_injected(&mut self, info: BarrierInfo) {
+        if let Some(tx) = self.injected.take() {
+            tx.send(info).ok();
         }
     }
 
diff --git a/src/meta/src/barrier/progress.rs b/src/meta/src/barrier/progress.rs
index d706aa5bf47fe..d484e471f4a31 100644
--- a/src/meta/src/barrier/progress.rs
+++ b/src/meta/src/barrier/progress.rs
@@ -27,7 +27,6 @@ use super::command::CommandContext;
 use super::notifier::Notifier;
 use crate::barrier::Command;
 use crate::model::ActorId;
-use crate::storage::MetaStore;
 
 type CreateMviewEpoch = Epoch;
 type ConsumedRows = u64;
@@ -142,9 +141,9 @@ impl Progress {
 }
 
 /// The command tracking by the [`CreateMviewProgressTracker`].
-pub(super) struct TrackingCommand<S: MetaStore> {
+pub(super) struct TrackingCommand {
     /// The context of the command.
-    pub context: Arc<CommandContext<S>>,
+    pub context: Arc<CommandContext>,
 
     /// Should be called when the command is finished.
     pub notifiers: Vec<Notifier>,
@@ -152,15 +151,15 @@ pub(super) struct TrackingCommand<S: MetaStore> {
 
 /// Track the progress of all creating mviews. When creation is done, `notify_finished` will be
 /// called on registered notifiers.
-pub(super) struct CreateMviewProgressTracker<S: MetaStore> {
+pub(super) struct CreateMviewProgressTracker {
     /// Progress of the create-mview DDL indicated by the epoch.
-    progress_map: HashMap<CreateMviewEpoch, (Progress, TrackingCommand<S>)>,
+    progress_map: HashMap<CreateMviewEpoch, (Progress, TrackingCommand)>,
 
     /// Find the epoch of the create-mview DDL by the actor containing the chain node.
     actor_map: HashMap<ActorId, CreateMviewEpoch>,
 }
 
-impl<S: MetaStore> CreateMviewProgressTracker<S> {
+impl CreateMviewProgressTracker {
     pub fn new() -> Self {
         Self {
             progress_map: Default::default(),
@@ -185,7 +184,7 @@ impl<S: MetaStore> CreateMviewProgressTracker<S> {
     pub fn find_cancelled_command(
         &mut self,
         actors_to_cancel: HashSet<ActorId>,
-    ) -> Option<TrackingCommand<S>> {
+    ) -> Option<TrackingCommand> {
         let epochs = actors_to_cancel
             .into_iter()
             .map(|actor_id| self.actor_map.get(&actor_id))
@@ -205,9 +204,9 @@ impl<S: MetaStore> CreateMviewProgressTracker<S> {
     /// If the actors to track is empty, return the given command as it can be finished immediately.
     pub fn add(
         &mut self,
-        command: TrackingCommand<S>,
+        command: TrackingCommand,
         version_stats: &HummockVersionStats,
-    ) -> Option<TrackingCommand<S>> {
+    ) -> Option<TrackingCommand> {
         let actors = command.context.actors_to_track();
         if actors.is_empty() {
             // The command can be finished immediately.
@@ -279,7 +278,7 @@ impl<S: MetaStore> CreateMviewProgressTracker<S> {
         &mut self,
         progress: &CreateMviewProgress,
         version_stats: &HummockVersionStats,
-    ) -> Option<TrackingCommand<S>> {
+    ) -> Option<TrackingCommand> {
         let actor = progress.chain_actor_id;
         let Some(epoch) = self.actor_map.get(&actor).copied() else {
             panic!(
diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs
index a049fe04f8877..bce901cd6f459 100644
--- a/src/meta/src/barrier/recovery.rs
+++ b/src/meta/src/barrier/recovery.rs
@@ -19,11 +19,11 @@ use std::time::{Duration, Instant};
 use futures::future::try_join_all;
 use itertools::Itertools;
 use risingwave_pb::common::ActorInfo;
+use risingwave_pb::meta::PausedReason;
 use risingwave_pb::stream_plan::barrier::{BarrierKind, Mutation};
 use risingwave_pb::stream_plan::AddMutation;
 use risingwave_pb::stream_service::{
-    BarrierCompleteResponse, BroadcastActorInfoTableRequest, BuildActorsRequest,
-    ForceStopActorsRequest, UpdateActorsRequest,
+    BroadcastActorInfoTableRequest, BuildActorsRequest, ForceStopActorsRequest, UpdateActorsRequest,
 };
 use tokio_retry::strategy::{jitter, ExponentialBackoff};
 use tracing::{debug, warn, Instrument};
@@ -34,15 +34,11 @@ use crate::barrier::command::CommandContext;
 use crate::barrier::info::BarrierActorInfo;
 use crate::barrier::{CheckpointControl, Command, GlobalBarrierManager};
 use crate::manager::WorkerId;
-use crate::model::MigrationPlan;
-use crate::storage::MetaStore;
+use crate::model::{BarrierManagerState, MigrationPlan};
 use crate::stream::build_actor_connector_splits;
 use crate::MetaResult;
 
-impl<S> GlobalBarrierManager<S>
-where
-    S: MetaStore,
-{
+impl GlobalBarrierManager {
     // Retry base interval in milliseconds.
     const RECOVERY_RETRY_BASE_INTERVAL: u64 = 20;
     // Retry max interval.
@@ -106,8 +102,16 @@ where
 
     /// Recovery the whole cluster from the latest epoch.
     ///
-    /// Returns the new epoch after recovery.
-    pub(crate) async fn recovery(&self, prev_epoch: TracedEpoch) -> TracedEpoch {
+    /// If `paused_reason` is `Some`, all data sources (including connectors and DMLs) will be
+    /// immediately paused after recovery, until the user manually resume them either by restarting
+    /// the cluster or `risectl` command. Used for debugging purpose.
+    ///
+    /// Returns the new state of the barrier manager after recovery.
+    pub(crate) async fn recovery(
+        &self,
+        prev_epoch: TracedEpoch,
+        paused_reason: Option<PausedReason>,
+    ) -> BarrierManagerState {
         // Mark blocked and abort buffered schedules, they might be dirty already.
         self.scheduled_barriers
             .abort_and_mark_blocked("cluster is under recovering")
@@ -123,9 +127,10 @@ where
         // We take retry into consideration because this is the latency user sees for a cluster to
         // get recovered.
         let recovery_timer = self.metrics.recovery_latency.start_timer();
-        let (new_epoch, _responses) = tokio_retry::Retry::spawn(retry_strategy, || {
+
+        let state = tokio_retry::Retry::spawn(retry_strategy, || {
             async {
-                let recovery_result: MetaResult<(TracedEpoch, Vec<BarrierCompleteResponse>)> = try {
+                let recovery_result: MetaResult<_> = try {
                     // Resolve actor info for recovery. If there's no actor to recover, most of the
                     // following steps will be no-op, while the compute nodes will still be reset.
                     let mut info = self.resolve_actor_info_for_recovery().await;
@@ -158,6 +163,7 @@ where
                         actor_dispatchers: Default::default(),
                         added_actors: Default::default(),
                         actor_splits: build_actor_connector_splits(&source_split_assignments),
+                        pause: paused_reason.is_some(),
                     })));
 
                     // Use a different `curr_epoch` for each recovery attempt.
@@ -170,6 +176,7 @@ where
                         info,
                         prev_epoch.clone(),
                         new_epoch.clone(),
+                        paused_reason,
                         command,
                         BarrierKind::Initial,
                         self.source_manager.clone(),
@@ -205,7 +212,9 @@ where
                             Err(err)
                         }
                     };
-                    res?
+                    let (new_epoch, _) = res?;
+
+                    BarrierManagerState::new(new_epoch, command_ctx.next_paused_reason())
                 };
                 if recovery_result.is_err() {
                     self.metrics.recovery_failure_cnt.inc();
@@ -216,11 +225,17 @@ where
         })
         .await
         .expect("Retry until recovery success.");
+
         recovery_timer.observe_duration();
         self.scheduled_barriers.mark_ready().await;
-        tracing::info!("recovery success");
 
-        new_epoch
+        tracing::info!(
+            epoch = state.in_flight_prev_epoch().value().0,
+            paused = ?state.paused_reason(),
+            "recovery success"
+        );
+
+        state
     }
 
     /// Migrate actors in expired CNs to newly joined ones, return true if any actor is migrated.
@@ -238,7 +253,7 @@ where
             debug!("no expired workers, skipping.");
             return Ok(false);
         }
-        let migration_plan = self.generate_migration_plan(info, expired_workers).await?;
+        let migration_plan = self.generate_migration_plan(expired_workers).await?;
         // 2. start to migrate fragment one-by-one.
         self.fragment_manager
             .migrate_fragment_actors(&migration_plan)
@@ -254,18 +269,25 @@ where
     /// in-used parallel unit to a new one.
     async fn generate_migration_plan(
         &self,
-        info: &BarrierActorInfo,
         expired_workers: HashSet<WorkerId>,
     ) -> MetaResult<MigrationPlan> {
         let mut cached_plan = MigrationPlan::get(self.env.meta_store()).await?;
 
         let all_worker_parallel_units = self.fragment_manager.all_worker_parallel_units().await;
 
-        let mut to_migrate_parallel_units: BTreeSet<_> = all_worker_parallel_units
-            .iter()
-            .filter(|(worker, _)| expired_workers.contains(worker))
-            .flat_map(|(_, pu)| pu.iter().cloned())
+        let (expired_inuse_workers, inuse_workers): (Vec<_>, Vec<_>) = all_worker_parallel_units
+            .into_iter()
+            .partition(|(worker, _)| expired_workers.contains(worker));
+
+        let mut to_migrate_parallel_units: BTreeSet<_> = expired_inuse_workers
+            .into_iter()
+            .flat_map(|(_, pu)| pu.into_iter())
             .collect();
+        let mut inuse_parallel_units: HashSet<_> = inuse_workers
+            .into_iter()
+            .flat_map(|(_, pu)| pu.into_iter())
+            .collect();
+
         cached_plan.parallel_unit_plan.retain(|from, to| {
             if to_migrate_parallel_units.contains(from) {
                 if !to_migrate_parallel_units.contains(&to.id) {
@@ -278,6 +300,7 @@ where
             false
         });
         to_migrate_parallel_units.retain(|id| !cached_plan.parallel_unit_plan.contains_key(id));
+        inuse_parallel_units.extend(cached_plan.parallel_unit_plan.values().map(|pu| pu.id));
 
         if to_migrate_parallel_units.is_empty() {
             // all expired parallel units are already in migration plan.
@@ -294,33 +317,21 @@ where
         let start = Instant::now();
         // if in-used expire parallel units are not empty, should wait for newly joined worker.
         'discovery: while !to_migrate_parallel_units.is_empty() {
-            let current_nodes = self
+            let mut new_parallel_units = self
                 .cluster_manager
-                .list_active_streaming_compute_nodes()
+                .list_active_streaming_parallel_units()
                 .await;
-            let new_nodes = current_nodes
-                .into_iter()
-                .filter(|node| {
-                    !info.actor_map.contains_key(&node.id)
-                        && !cached_plan
-                            .parallel_unit_plan
-                            .values()
-                            .map(|pu| pu.worker_node_id)
-                            .contains(&node.id)
-                })
-                .collect_vec();
-            if !new_nodes.is_empty() {
-                debug!("new workers joined: {:#?}", new_nodes);
-                let target_parallel_units = new_nodes
-                    .into_iter()
-                    .flat_map(|node| node.parallel_units)
-                    .collect_vec();
-                for target_parallel_unit in target_parallel_units {
+            new_parallel_units.retain(|pu| !inuse_parallel_units.contains(&pu.id));
+
+            if !new_parallel_units.is_empty() {
+                debug!("new parallel units found: {:#?}", new_parallel_units);
+                for target_parallel_unit in new_parallel_units {
                     if let Some(from) = to_migrate_parallel_units.pop() {
                         debug!(
                             "plan to migrate from parallel unit {} to {}",
                             from, target_parallel_unit.id
                         );
+                        inuse_parallel_units.insert(target_parallel_unit.id);
                         cached_plan
                             .parallel_unit_plan
                             .insert(from, target_parallel_unit);
diff --git a/src/meta/src/barrier/schedule.rs b/src/meta/src/barrier/schedule.rs
index 1c93665002ac3..7c9fefd15606b 100644
--- a/src/meta/src/barrier/schedule.rs
+++ b/src/meta/src/barrier/schedule.rs
@@ -21,13 +21,13 @@ use std::time::Instant;
 use anyhow::anyhow;
 use risingwave_common::catalog::TableId;
 use risingwave_pb::hummock::HummockSnapshot;
+use risingwave_pb::meta::PausedReason;
 use tokio::sync::{oneshot, watch, RwLock};
 
-use super::notifier::Notifier;
+use super::notifier::{BarrierInfo, Notifier};
 use super::{Command, Scheduled};
 use crate::hummock::HummockManagerRef;
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::MetaStore;
 use crate::{MetaError, MetaResult};
 
 /// A queue for scheduling barriers.
@@ -121,18 +121,18 @@ impl Inner {
 /// The sender side of the barrier scheduling queue.
 /// Can be cloned and held by other managers to schedule and run barriers.
 #[derive(Clone)]
-pub struct BarrierScheduler<S: MetaStore> {
+pub struct BarrierScheduler {
     inner: Arc<Inner>,
 
     /// Used for getting the latest snapshot after `FLUSH`.
-    hummock_manager: HummockManagerRef<S>,
+    hummock_manager: HummockManagerRef,
 }
 
-impl<S: MetaStore> BarrierScheduler<S> {
+impl BarrierScheduler {
     /// Create a pair of [`BarrierScheduler`] and [`ScheduledBarriers`], for scheduling barriers
     /// from different managers, and executing them in the barrier manager, respectively.
     pub fn new_pair(
-        hummock_manager: HummockManagerRef<S>,
+        hummock_manager: HummockManagerRef,
         metrics: Arc<MetaMetrics>,
         checkpoint_frequency: usize,
     ) -> (Self, ScheduledBarriers) {
@@ -237,42 +237,41 @@ impl<S: MetaStore> BarrierScheduler<S> {
     /// Run multiple commands and return when they're all completely finished. It's ensured that
     /// multiple commands are executed continuously.
     ///
+    /// Returns the barrier info of each command.
+    ///
     /// TODO: atomicity of multiple commands is not guaranteed.
-    pub async fn run_multiple_commands(&self, commands: Vec<Command>) -> MetaResult<()> {
-        struct Context {
-            collect_rx: oneshot::Receiver<MetaResult<()>>,
-            finish_rx: oneshot::Receiver<()>,
-        }
-
+    async fn run_multiple_commands(&self, commands: Vec<Command>) -> MetaResult<Vec<BarrierInfo>> {
         let mut contexts = Vec::with_capacity(commands.len());
         let mut scheduleds = Vec::with_capacity(commands.len());
 
         for command in commands {
+            let (injected_tx, injected_rx) = oneshot::channel();
             let (collect_tx, collect_rx) = oneshot::channel();
             let (finish_tx, finish_rx) = oneshot::channel();
 
-            contexts.push(Context {
-                collect_rx,
-                finish_rx,
-            });
+            contexts.push((injected_rx, collect_rx, finish_rx));
             scheduleds.push(self.inner.new_scheduled(
                 command.need_checkpoint(),
                 command,
                 once(Notifier {
+                    injected: Some(injected_tx),
                     collected: Some(collect_tx),
                     finished: Some(finish_tx),
-                    ..Default::default()
                 }),
             ));
         }
 
         self.push(scheduleds).await?;
 
-        for Context {
-            collect_rx,
-            finish_rx,
-        } in contexts
-        {
+        let mut infos = Vec::with_capacity(contexts.len());
+
+        for (injected_rx, collect_rx, finish_rx) in contexts {
+            // Wait for this command to be injected, and record the result.
+            let info = injected_rx
+                .await
+                .map_err(|e| anyhow!("failed to inject barrier: {}", e))?;
+            infos.push(info);
+
             // Throw the error if it occurs when collecting this barrier.
             collect_rx
                 .await
@@ -284,19 +283,33 @@ impl<S: MetaStore> BarrierScheduler<S> {
                 .map_err(|e| anyhow!("failed to finish command: {}", e))?;
         }
 
-        Ok(())
+        Ok(infos)
     }
 
     /// Run a command with a `Pause` command before and `Resume` command after it. Used for
     /// configuration change.
-    pub async fn run_command_with_paused(&self, command: Command) -> MetaResult<()> {
-        self.run_multiple_commands(vec![Command::pause(), command, Command::resume()])
-            .await
+    ///
+    /// Returns the barrier info of the actual command.
+    pub async fn run_config_change_command_with_pause(
+        &self,
+        command: Command,
+    ) -> MetaResult<BarrierInfo> {
+        self.run_multiple_commands(vec![
+            Command::pause(PausedReason::ConfigChange),
+            command,
+            Command::resume(PausedReason::ConfigChange),
+        ])
+        .await
+        .map(|i| i[1])
     }
 
     /// Run a command and return when it's completely finished.
-    pub async fn run_command(&self, command: Command) -> MetaResult<()> {
-        self.run_multiple_commands(vec![command]).await
+    ///
+    /// Returns the barrier info of the actual command.
+    pub async fn run_command(&self, command: Command) -> MetaResult<BarrierInfo> {
+        self.run_multiple_commands(vec![command])
+            .await
+            .map(|i| i[0])
     }
 
     /// Flush means waiting for the next barrier to collect.
diff --git a/src/meta/src/dashboard/mod.rs b/src/meta/src/dashboard/mod.rs
index 93704a3dc81e0..a08eaa70cf515 100644
--- a/src/meta/src/dashboard/mod.rs
+++ b/src/meta/src/dashboard/mod.rs
@@ -35,22 +35,22 @@ use tower_http::cors::{self, CorsLayer};
 use tower_http::services::ServeDir;
 
 use crate::manager::{ClusterManagerRef, FragmentManagerRef};
-use crate::storage::MetaStore;
+use crate::storage::MetaStoreRef;
 
 #[derive(Clone)]
-pub struct DashboardService<S: MetaStore> {
+pub struct DashboardService {
     pub dashboard_addr: SocketAddr,
     pub prometheus_endpoint: Option<String>,
     pub prometheus_client: Option<prometheus_http_query::Client>,
-    pub cluster_manager: ClusterManagerRef<S>,
-    pub fragment_manager: FragmentManagerRef<S>,
+    pub cluster_manager: ClusterManagerRef,
+    pub fragment_manager: FragmentManagerRef,
     pub compute_clients: ComputeClientPool,
 
     // TODO: replace with catalog manager.
-    pub meta_store: Arc<S>,
+    pub meta_store: MetaStoreRef,
 }
 
-pub type Service<S> = Arc<DashboardService<S>>;
+pub type Service = Arc<DashboardService>;
 
 pub(super) mod handlers {
     use anyhow::Context;
@@ -66,6 +66,7 @@ pub(super) mod handlers {
     use super::*;
     use crate::manager::WorkerId;
     use crate::model::TableFragments;
+    use crate::storage::MetaStoreRef;
 
     pub struct DashboardError(anyhow::Error);
     pub type Result<T> = std::result::Result<T, DashboardError>;
@@ -86,9 +87,9 @@ pub(super) mod handlers {
         }
     }
 
-    pub async fn list_clusters<S: MetaStore>(
+    pub async fn list_clusters(
         Path(ty): Path<i32>,
-        Extension(srv): Extension<Service<S>>,
+        Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<WorkerNode>>> {
         use risingwave_pb::common::WorkerType;
         let mut result = srv
@@ -104,8 +105,8 @@ pub(super) mod handlers {
         Ok(result.into())
     }
 
-    async fn list_table_catalogs_inner<S: MetaStore>(
-        meta_store: &S,
+    async fn list_table_catalogs_inner(
+        meta_store: &MetaStoreRef,
         table_type: TableType,
     ) -> Result<Json<Vec<Table>>> {
         use crate::model::MetadataModel;
@@ -120,50 +121,42 @@ pub(super) mod handlers {
         Ok(Json(results))
     }
 
-    pub async fn list_materialized_views<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
+    pub async fn list_materialized_views(
+        Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<Table>>> {
-        list_table_catalogs_inner(&*srv.meta_store, TableType::MaterializedView).await
+        list_table_catalogs_inner(&srv.meta_store, TableType::MaterializedView).await
     }
 
-    pub async fn list_tables<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
-    ) -> Result<Json<Vec<Table>>> {
-        list_table_catalogs_inner(&*srv.meta_store, TableType::Table).await
+    pub async fn list_tables(Extension(srv): Extension<Service>) -> Result<Json<Vec<Table>>> {
+        list_table_catalogs_inner(&srv.meta_store, TableType::Table).await
     }
 
-    pub async fn list_indexes<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
-    ) -> Result<Json<Vec<Table>>> {
-        list_table_catalogs_inner(&*srv.meta_store, TableType::Index).await
+    pub async fn list_indexes(Extension(srv): Extension<Service>) -> Result<Json<Vec<Table>>> {
+        list_table_catalogs_inner(&srv.meta_store, TableType::Index).await
     }
 
-    pub async fn list_internal_tables<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
+    pub async fn list_internal_tables(
+        Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<Table>>> {
-        list_table_catalogs_inner(&*srv.meta_store, TableType::Internal).await
+        list_table_catalogs_inner(&srv.meta_store, TableType::Internal).await
     }
 
-    pub async fn list_sources<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
-    ) -> Result<Json<Vec<Source>>> {
+    pub async fn list_sources(Extension(srv): Extension<Service>) -> Result<Json<Vec<Source>>> {
         use crate::model::MetadataModel;
 
-        let sources = Source::list(&*srv.meta_store).await.map_err(err)?;
+        let sources = Source::list(&srv.meta_store).await.map_err(err)?;
         Ok(Json(sources))
     }
 
-    pub async fn list_sinks<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
-    ) -> Result<Json<Vec<Sink>>> {
+    pub async fn list_sinks(Extension(srv): Extension<Service>) -> Result<Json<Vec<Sink>>> {
         use crate::model::MetadataModel;
 
-        let sinks = Sink::list(&*srv.meta_store).await.map_err(err)?;
+        let sinks = Sink::list(&srv.meta_store).await.map_err(err)?;
         Ok(Json(sinks))
     }
 
-    pub async fn list_actors<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
+    pub async fn list_actors(
+        Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<ActorLocation>>> {
         let mut node_actors = srv.fragment_manager.all_node_actors(true).await;
         let nodes = srv
@@ -181,12 +174,12 @@ pub(super) mod handlers {
         Ok(Json(actors))
     }
 
-    pub async fn list_fragments<S: MetaStore>(
-        Extension(srv): Extension<Service<S>>,
+    pub async fn list_fragments(
+        Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<PbTableFragments>>> {
         use crate::model::MetadataModel;
 
-        let table_fragments = TableFragments::list(&*srv.meta_store)
+        let table_fragments = TableFragments::list(&srv.meta_store)
             .await
             .map_err(err)?
             .into_iter()
@@ -195,9 +188,9 @@ pub(super) mod handlers {
         Ok(Json(table_fragments))
     }
 
-    pub async fn dump_await_tree<S: MetaStore>(
+    pub async fn dump_await_tree(
         Path(worker_id): Path<WorkerId>,
-        Extension(srv): Extension<Service<S>>,
+        Extension(srv): Extension<Service>,
     ) -> Result<Json<StackTraceResponse>> {
         let worker_node = srv
             .cluster_manager
@@ -215,10 +208,7 @@ pub(super) mod handlers {
     }
 }
 
-impl<S> DashboardService<S>
-where
-    S: MetaStore,
-{
+impl DashboardService {
     pub async fn serve(self, ui_path: Option<String>) -> Result<()> {
         use handlers::*;
         let srv = Arc::new(self);
@@ -228,24 +218,21 @@ where
             .allow_methods(vec![Method::GET]);
 
         let api_router = Router::new()
-            .route("/clusters/:ty", get(list_clusters::<S>))
-            .route("/actors", get(list_actors::<S>))
-            .route("/fragments2", get(list_fragments::<S>))
-            .route("/materialized_views", get(list_materialized_views::<S>))
-            .route("/tables", get(list_tables::<S>))
-            .route("/indexes", get(list_indexes::<S>))
-            .route("/internal_tables", get(list_internal_tables::<S>))
-            .route("/sources", get(list_sources::<S>))
-            .route("/sinks", get(list_sinks::<S>))
-            .route(
-                "/metrics/cluster",
-                get(prometheus::list_prometheus_cluster::<S>),
-            )
+            .route("/clusters/:ty", get(list_clusters))
+            .route("/actors", get(list_actors))
+            .route("/fragments2", get(list_fragments))
+            .route("/materialized_views", get(list_materialized_views))
+            .route("/tables", get(list_tables))
+            .route("/indexes", get(list_indexes))
+            .route("/internal_tables", get(list_internal_tables))
+            .route("/sources", get(list_sources))
+            .route("/sinks", get(list_sinks))
+            .route("/metrics/cluster", get(prometheus::list_prometheus_cluster))
             .route(
                 "/metrics/actor/back_pressures",
-                get(prometheus::list_prometheus_actor_back_pressure::<S>),
+                get(prometheus::list_prometheus_actor_back_pressure),
             )
-            .route("/monitor/await_tree/:worker_id", get(dump_await_tree::<S>))
+            .route("/monitor/await_tree/:worker_id", get(dump_await_tree))
             .layer(
                 ServiceBuilder::new()
                     .layer(AddExtensionLayer::new(srv.clone()))
diff --git a/src/meta/src/dashboard/prometheus.rs b/src/meta/src/dashboard/prometheus.rs
index e40399b447f15..49431a29afd65 100644
--- a/src/meta/src/dashboard/prometheus.rs
+++ b/src/meta/src/dashboard/prometheus.rs
@@ -22,7 +22,6 @@ use serde::Serialize;
 
 use super::handlers::{err, DashboardError};
 use super::Service;
-use crate::storage::MetaStore;
 
 #[derive(Serialize, Debug)]
 pub struct PrometheusSample {
@@ -63,8 +62,8 @@ pub struct ClusterMetrics {
 
 pub type Result<T> = std::result::Result<T, DashboardError>;
 
-pub async fn list_prometheus_cluster<S: MetaStore>(
-    Extension(srv): Extension<Service<S>>,
+pub async fn list_prometheus_cluster(
+    Extension(srv): Extension<Service>,
 ) -> Result<Json<ClusterMetrics>> {
     if let Some(ref client) = srv.prometheus_client {
         // assume job_name is one of compute, meta, frontend
@@ -130,8 +129,8 @@ pub async fn list_prometheus_cluster<S: MetaStore>(
 pub struct ActorBackPressure {
     output_buffer_blocking_duration: Vec<PrometheusVector>,
 }
-pub async fn list_prometheus_actor_back_pressure<S: MetaStore>(
-    Extension(srv): Extension<Service<S>>,
+pub async fn list_prometheus_actor_back_pressure(
+    Extension(srv): Extension<Service>,
 ) -> Result<Json<ActorBackPressure>> {
     if let Some(ref client) = srv.prometheus_client {
         let now = SystemTime::now();
diff --git a/src/meta/src/hummock/compaction/level_selector.rs b/src/meta/src/hummock/compaction/level_selector.rs
index f36faab140797..893dffd79d6bc 100644
--- a/src/meta/src/hummock/compaction/level_selector.rs
+++ b/src/meta/src/hummock/compaction/level_selector.rs
@@ -26,8 +26,8 @@ use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::{compact_task, CompactionConfig, LevelType};
 
 use super::picker::{
-    SpaceReclaimCompactionPicker, SpaceReclaimPickerState, TtlPickerState,
-    TtlReclaimCompactionPicker,
+    CompactionTaskValidator, IntraCompactionPicker, SpaceReclaimCompactionPicker,
+    SpaceReclaimPickerState, TtlPickerState, TtlReclaimCompactionPicker,
 };
 use super::{
     create_compaction_task, LevelCompactionPicker, ManualCompactionOption, ManualCompactionPicker,
@@ -44,6 +44,23 @@ use crate::rpc::metrics::MetaMetrics;
 
 pub const SCORE_BASE: u64 = 100;
 
+#[derive(Debug, Default, Clone)]
+pub enum PickerType {
+    Tier,
+    Intra,
+    ToBase,
+    #[default]
+    BottomLevel,
+}
+
+#[derive(Default, Debug)]
+pub struct PickerInfo {
+    score: u64,
+    select_level: usize,
+    target_level: usize,
+    picker_type: PickerType,
+}
+
 pub trait LevelSelector: Sync + Send {
     fn pick_compaction(
         &mut self,
@@ -71,7 +88,7 @@ pub struct SelectContext {
     // size of the files in  `base_level` reaches its capacity, we will place data in a higher
     // level, which equals to `base_level -= 1;`.
     pub base_level: usize,
-    pub score_levels: Vec<(u64, usize, usize)>,
+    pub score_levels: Vec<PickerInfo>,
 }
 
 pub struct DynamicLevelSelectorCore {
@@ -92,35 +109,41 @@ impl DynamicLevelSelectorCore {
 
     fn create_compaction_picker(
         &self,
-        select_level: usize,
-        target_level: usize,
+        picker_info: &PickerInfo,
         overlap_strategy: Arc<dyn OverlapStrategy>,
+        compaction_task_validator: Arc<CompactionTaskValidator>,
     ) -> Box<dyn CompactionPicker> {
-        if select_level == 0 {
-            if target_level == 0 {
-                Box::new(TierCompactionPicker::new(self.config.clone()))
-            } else {
-                Box::new(LevelCompactionPicker::new(
-                    target_level,
-                    self.config.clone(),
+        match picker_info.picker_type {
+            PickerType::Tier => Box::new(TierCompactionPicker::new_with_validator(
+                self.config.clone(),
+                compaction_task_validator,
+            )),
+            PickerType::ToBase => Box::new(LevelCompactionPicker::new_with_validator(
+                picker_info.target_level,
+                self.config.clone(),
+                compaction_task_validator,
+            )),
+            PickerType::Intra => Box::new(IntraCompactionPicker::new_with_validator(
+                self.config.clone(),
+                compaction_task_validator,
+            )),
+            PickerType::BottomLevel => {
+                assert_eq!(picker_info.select_level + 1, picker_info.target_level);
+                Box::new(MinOverlappingPicker::new(
+                    picker_info.select_level,
+                    picker_info.target_level,
+                    self.config.max_bytes_for_level_base,
+                    self.config.split_by_state_table,
+                    overlap_strategy,
                 ))
             }
-        } else {
-            assert_eq!(select_level + 1, target_level);
-            Box::new(MinOverlappingPicker::new(
-                select_level,
-                target_level,
-                self.config.max_bytes_for_level_base,
-                self.config.split_by_state_table,
-                overlap_strategy,
-            ))
         }
     }
 
     // TODO: calculate this scores in apply compact result.
     /// `calculate_level_base_size` calculate base level and the base size of LSM tree build for
     /// current dataset. In other words,  `level_max_bytes` is our compaction goal which shall
-    /// reach. This algorithm refers to the implementation in  `</>https://github.com/facebook/rocksdb/blob/v7.2.2/db/version_set.cc#L3706</>`
+    /// reach. This algorithm refers to the implementation in  [`https://github.com/facebook/rocksdb/blob/v7.2.2/db/version_set.cc#L3706`]
     pub fn calculate_level_base_size(&self, levels: &Levels) -> SelectContext {
         let mut first_non_empty_level = 0;
         let mut max_level_size = 0;
@@ -212,8 +235,12 @@ impl DynamicLevelSelectorCore {
                     std::cmp::min(idle_file_count, overlapping_file_count) as u64 * SCORE_BASE
                         / self.config.level0_tier_compact_file_number;
                 // Reduce the level num of l0 overlapping sub_level
-                ctx.score_levels
-                    .push((std::cmp::max(l0_overlapping_score, SCORE_BASE + 1), 0, 0));
+                ctx.score_levels.push(PickerInfo {
+                    score: std::cmp::max(l0_overlapping_score, SCORE_BASE + 1),
+                    select_level: 0,
+                    target_level: 0,
+                    picker_type: PickerType::Tier,
+                })
             }
 
             // The read query at the non-overlapping level only selects ssts that match the query
@@ -249,8 +276,24 @@ impl DynamicLevelSelectorCore {
             };
 
             // Reduce the level num of l0 non-overlapping sub_level
-            ctx.score_levels
-                .push((non_overlapping_score, 0, ctx.base_level));
+            ctx.score_levels.push({
+                PickerInfo {
+                    score: non_overlapping_score,
+                    select_level: 0,
+                    target_level: ctx.base_level,
+                    picker_type: PickerType::ToBase,
+                }
+            });
+
+            // FIXME: more accurate score calculation algorithm will be introduced (#11903)
+            ctx.score_levels.push({
+                PickerInfo {
+                    score: non_overlapping_score,
+                    select_level: 0,
+                    target_level: 0,
+                    picker_type: PickerType::Intra,
+                }
+            });
         }
 
         // The bottommost level can not be input level.
@@ -270,23 +313,30 @@ impl DynamicLevelSelectorCore {
             if total_size == 0 {
                 continue;
             }
-            ctx.score_levels.push((
-                total_size * SCORE_BASE / ctx.level_max_bytes[level_idx],
-                level_idx,
-                level_idx + 1,
-            ));
+
+            ctx.score_levels.push({
+                PickerInfo {
+                    score: total_size * SCORE_BASE / ctx.level_max_bytes[level_idx],
+                    select_level: level_idx,
+                    target_level: level_idx + 1,
+                    picker_type: PickerType::BottomLevel,
+                }
+            });
         }
 
         // sort reverse to pick the largest one.
-        ctx.score_levels
-            .sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.2.cmp(&b.2)));
+        ctx.score_levels.sort_by(|a, b| {
+            b.score
+                .cmp(&a.score)
+                .then_with(|| a.target_level.cmp(&b.target_level))
+        });
         ctx
     }
 
     /// `compact_pending_bytes_needed` calculates the number of compact bytes needed to balance the
     /// LSM Tree from the current state of each level in the LSM Tree in combination with
     /// `compaction_config`
-    /// This algorithm refers to the implementation in  `</>https://github.com/facebook/rocksdb/blob/main/db/version_set.cc#L3141</>`
+    /// This algorithm refers to the implementation in  [`https://github.com/facebook/rocksdb/blob/main/db/version_set.cc#L3141`]
     pub fn compact_pending_bytes_needed(&self, levels: &Levels) -> u64 {
         let ctx = self.calculate_level_base_size(levels);
         self.compact_pending_bytes_needed_with_ctx(levels, &ctx)
@@ -374,15 +424,20 @@ impl LevelSelector for DynamicLevelSelector {
         let overlap_strategy =
             create_overlap_strategy(compaction_group.compaction_config.compaction_mode());
         let ctx = dynamic_level_core.get_priority_levels(levels, level_handlers);
-        for (score, select_level, target_level) in ctx.score_levels {
-            if score <= SCORE_BASE {
+        // TODO: Determine which rule to enable by write limit
+        let compaction_task_validator = Arc::new(CompactionTaskValidator::new(
+            compaction_group.compaction_config.clone(),
+        ));
+        for picker_info in &ctx.score_levels {
+            if picker_info.score <= SCORE_BASE {
                 return None;
             }
             let mut picker = dynamic_level_core.create_compaction_picker(
-                select_level,
-                target_level,
+                picker_info,
                 overlap_strategy.clone(),
+                compaction_task_validator.clone(),
             );
+
             let mut stats = LocalPickerStatistic::default();
             if let Some(ret) = picker.pick_compaction(levels, level_handlers, &mut stats) {
                 ret.add_pending_task(task_id, level_handlers);
@@ -393,9 +448,11 @@ impl LevelSelector for DynamicLevelSelector {
                     self.task_type(),
                 ));
             }
-            selector_stats
-                .skip_picker
-                .push((select_level, target_level, stats));
+            selector_stats.skip_picker.push((
+                picker_info.select_level,
+                picker_info.target_level,
+                stats,
+            ));
         }
         None
     }
diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs
index 97443f3bc7a96..a9f8d95457ccf 100644
--- a/src/meta/src/hummock/compaction/mod.rs
+++ b/src/meta/src/hummock/compaction/mod.rs
@@ -247,25 +247,25 @@ impl LocalSelectorStatistic {
                 metrics
                     .compact_skip_frequency
                     .with_label_values(&[level_label.as_str(), "write-amp"])
-                    .inc_by(stats.skip_by_write_amp_limit);
+                    .inc();
             }
             if stats.skip_by_count_limit > 0 {
                 metrics
                     .compact_skip_frequency
                     .with_label_values(&[level_label.as_str(), "count"])
-                    .inc_by(stats.skip_by_count_limit);
+                    .inc();
             }
             if stats.skip_by_pending_files > 0 {
                 metrics
                     .compact_skip_frequency
                     .with_label_values(&[level_label.as_str(), "pending-files"])
-                    .inc_by(stats.skip_by_pending_files);
+                    .inc();
             }
             if stats.skip_by_overlapping > 0 {
                 metrics
                     .compact_skip_frequency
                     .with_label_values(&[level_label.as_str(), "overlapping"])
-                    .inc_by(stats.skip_by_overlapping);
+                    .inc();
             }
             metrics
                 .compact_skip_frequency
diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
index a15fab694ee86..c224fbfe6ce55 100644
--- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
@@ -20,7 +20,10 @@ use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::{CompactionConfig, InputLevel, Level, LevelType, OverlappingLevel};
 
 use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker;
-use super::{CompactionInput, CompactionPicker, LocalPickerStatistic};
+use super::{
+    CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic,
+    ValidationRuleType,
+};
 use crate::hummock::compaction::create_overlap_strategy;
 use crate::hummock::compaction::picker::TrivialMovePicker;
 use crate::hummock::level_handler::LevelHandler;
@@ -28,6 +31,7 @@ use crate::hummock::level_handler::LevelHandler;
 pub struct LevelCompactionPicker {
     target_level: usize,
     config: Arc<CompactionConfig>,
+    compaction_task_validator: Arc<CompactionTaskValidator>,
 }
 
 impl CompactionPicker for LevelCompactionPicker {
@@ -75,19 +79,29 @@ impl CompactionPicker for LevelCompactionPicker {
             return Some(ret);
         }
 
-        if let Some(ret) = self.pick_l0_intra(l0, &level_handlers[0], stats) {
-            return Some(ret);
-        }
-
-        self.pick_l0_trivial_move_file(l0, level_handlers, stats)
+        None
     }
 }
 
 impl LevelCompactionPicker {
+    #[cfg(test)]
     pub fn new(target_level: usize, config: Arc<CompactionConfig>) -> LevelCompactionPicker {
+        LevelCompactionPicker {
+            target_level,
+            compaction_task_validator: Arc::new(CompactionTaskValidator::new(config.clone())),
+            config,
+        }
+    }
+
+    pub fn new_with_validator(
+        target_level: usize,
+        config: Arc<CompactionConfig>,
+        compaction_task_validator: Arc<CompactionTaskValidator>,
+    ) -> LevelCompactionPicker {
         LevelCompactionPicker {
             target_level,
             config,
+            compaction_task_validator,
         }
     }
 
@@ -117,10 +131,10 @@ impl LevelCompactionPicker {
         level_handlers: &[LevelHandler],
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
+        // TODO: remove this
         let l0_size = l0.total_file_size - level_handlers[0].get_pending_file_size();
         let base_level_size = target_level.total_file_size
             - level_handlers[target_level.level_idx as usize].get_pending_file_size();
-
         if l0_size < base_level_size {
             stats.skip_by_write_amp_limit += 1;
             return None;
@@ -157,7 +171,7 @@ impl LevelCompactionPicker {
 
         let mut skip_by_pending = false;
         let mut input_levels = vec![];
-        let mut min_write_amp_meet = false;
+
         for input in l0_select_tables_vec {
             let l0_select_tables = input
                 .sstable_infos
@@ -184,16 +198,6 @@ impl LevelCompactionPicker {
                 continue;
             }
 
-            // The size of target level may be too large, we shall skip this compact task and wait
-            //  the data in base level compact to lower level.
-            if target_level_size > self.config.max_compaction_bytes && strict_check {
-                continue;
-            }
-
-            if input.total_file_size >= target_level_size {
-                min_write_amp_meet = true;
-            }
-
             input_levels.push((input, target_level_size, target_level_ssts));
         }
 
@@ -204,20 +208,7 @@ impl LevelCompactionPicker {
             return None;
         }
 
-        if !min_write_amp_meet && strict_check {
-            // If the write-amplification of all candidate task are large, we may hope to wait base
-            // level compact more data to lower level.  But if we skip all task, I'm
-            // afraid the data will be blocked in level0 and will be never compacted to base level.
-            // So we only allow one task exceed write-amplification-limit running in
-            // level0 to base-level.
-            return None;
-        }
-
         for (input, target_file_size, target_level_files) in input_levels {
-            if min_write_amp_meet && input.total_file_size < target_file_size {
-                continue;
-            }
-
             let mut select_level_inputs = input
                 .sstable_infos
                 .into_iter()
@@ -228,197 +219,32 @@ impl LevelCompactionPicker {
                 })
                 .collect_vec();
             select_level_inputs.reverse();
+            let target_file_count = target_level_files.len();
             select_level_inputs.push(InputLevel {
                 level_idx: target_level.level_idx,
                 level_type: target_level.level_type,
                 table_infos: target_level_files,
             });
-            return Some(CompactionInput {
+
+            let result = CompactionInput {
                 input_levels: select_level_inputs,
                 target_level: self.target_level,
-                target_sub_level_id: 0,
-            });
-        }
-        stats.skip_by_write_amp_limit += 1;
-        None
-    }
-
-    fn pick_l0_intra(
-        &self,
-        l0: &OverlappingLevel,
-        level_handler: &LevelHandler,
-        stats: &mut LocalPickerStatistic,
-    ) -> Option<CompactionInput> {
-        let overlap_strategy = create_overlap_strategy(self.config.compaction_mode());
-
-        for (idx, level) in l0.sub_levels.iter().enumerate() {
-            if level.level_type() != LevelType::Nonoverlapping
-                || level.total_file_size > self.config.sub_level_max_compaction_bytes
-            {
-                continue;
-            }
-
-            if level_handler.is_level_all_pending_compact(level) {
-                continue;
-            }
-
-            let max_compaction_bytes = std::cmp::min(
-                self.config.max_compaction_bytes,
-                self.config.sub_level_max_compaction_bytes,
-            );
-
-            let tier_sub_level_compact_level_count =
-                self.config.level0_sub_level_compact_level_count as usize;
-            let non_overlap_sub_level_picker = NonOverlapSubLevelPicker::new(
-                self.config.sub_level_max_compaction_bytes / 2,
-                max_compaction_bytes,
-                self.config.level0_sub_level_compact_level_count as usize,
-                self.config.level0_max_compact_file_number,
-                overlap_strategy.clone(),
-            );
-
-            let l0_select_tables_vec = non_overlap_sub_level_picker
-                .pick_l0_multi_non_overlap_level(&l0.sub_levels[idx..], level_handler);
-
-            if l0_select_tables_vec.is_empty() {
-                continue;
-            }
-
-            let mut skip_by_write_amp = false;
-            // Limit the number of selection levels for the non-overlapping
-            // sub_level at least level0_sub_level_compact_level_count
-            for (plan_index, input) in l0_select_tables_vec.into_iter().enumerate() {
-                if plan_index == 0
-                    && input.sstable_infos.len()
-                        < self.config.level0_sub_level_compact_level_count as usize
-                {
-                    // first plan level count smaller than limit
-                    break;
-                }
-
-                let mut max_level_size = 0;
-                for level_select_table in &input.sstable_infos {
-                    let level_select_size = level_select_table
-                        .iter()
-                        .map(|sst| sst.file_size)
-                        .sum::<u64>();
-
-                    max_level_size = std::cmp::max(max_level_size, level_select_size);
-                }
-
-                // This limitation would keep our write-amplification no more than
-                // ln(max_compaction_bytes/flush_level_bytes) /
-                // ln(self.config.level0_sub_level_compact_level_count/2) Here we only use half
-                // of level0_sub_level_compact_level_count just for convenient.
-                let is_write_amp_large =
-                    max_level_size * self.config.level0_sub_level_compact_level_count as u64 / 2
-                        >= input.total_file_size;
-
-                if (is_write_amp_large
-                    || input.sstable_infos.len() < tier_sub_level_compact_level_count)
-                    && input.total_file_count < self.config.level0_max_compact_file_number as usize
-                {
-                    skip_by_write_amp = true;
-                    continue;
-                }
-
-                let mut select_level_inputs = Vec::with_capacity(input.sstable_infos.len());
-                for level_select_sst in input.sstable_infos {
-                    if level_select_sst.is_empty() {
-                        continue;
-                    }
-                    select_level_inputs.push(InputLevel {
-                        level_idx: 0,
-                        level_type: LevelType::Nonoverlapping as i32,
-                        table_infos: level_select_sst,
-                    });
-                }
-                select_level_inputs.reverse();
-                return Some(CompactionInput {
-                    input_levels: select_level_inputs,
-                    target_level: 0,
-                    target_sub_level_id: level.sub_level_id,
-                });
-            }
-
-            if skip_by_write_amp {
-                stats.skip_by_write_amp_limit += 1;
-            }
-        }
-
-        None
-    }
-
-    fn pick_l0_trivial_move_file(
-        &self,
-        l0: &OverlappingLevel,
-        level_handlers: &[LevelHandler],
-        stats: &mut LocalPickerStatistic,
-    ) -> Option<CompactionInput> {
-        let overlap_strategy = create_overlap_strategy(self.config.compaction_mode());
-
-        for (idx, level) in l0.sub_levels.iter().enumerate() {
-            if level.level_type == LevelType::Overlapping as i32 || idx + 1 >= l0.sub_levels.len() {
-                continue;
-            }
-
-            if l0.sub_levels[idx + 1].level_type == LevelType::Overlapping as i32 {
-                continue;
-            }
-
-            let trivial_move_picker = TrivialMovePicker::new(0, 0, overlap_strategy.clone());
+                select_input_size: input.total_file_size,
+                target_input_size: target_file_size,
+                total_file_count: (input.total_file_count + target_file_count) as u64,
+                ..Default::default()
+            };
 
-            let select_sst = trivial_move_picker.pick_trivial_move_sst(
-                &l0.sub_levels[idx + 1].table_infos,
-                &level.table_infos,
-                level_handlers,
+            if !self.compaction_task_validator.valid_compact_task(
+                &result,
+                ValidationRuleType::ToBase,
                 stats,
-            );
-
-            // only pick tables for trivial move
-            if select_sst.is_none() {
+            ) && strict_check
+            {
                 continue;
             }
 
-            let select_sst = select_sst.unwrap();
-
-            // support trivial move cross multi sub_levels
-            let mut overlap = overlap_strategy.create_overlap_info();
-            overlap.update(&select_sst);
-
-            assert!(overlap
-                .check_multiple_overlap(&l0.sub_levels[idx].table_infos)
-                .is_empty());
-            let mut target_level_idx = idx;
-            while target_level_idx > 0 {
-                if l0.sub_levels[target_level_idx - 1].level_type
-                    != LevelType::Nonoverlapping as i32
-                    || !overlap
-                        .check_multiple_overlap(&l0.sub_levels[target_level_idx - 1].table_infos)
-                        .is_empty()
-                {
-                    break;
-                }
-                target_level_idx -= 1;
-            }
-
-            let input_levels = vec![
-                InputLevel {
-                    level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
-                    table_infos: vec![select_sst],
-                },
-                InputLevel {
-                    level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
-                    table_infos: vec![],
-                },
-            ];
-            return Some(CompactionInput {
-                input_levels,
-                target_level: 0,
-                target_sub_level_id: l0.sub_levels[target_level_idx].sub_level_id,
-            });
+            return Some(result);
         }
         None
     }
@@ -659,42 +485,6 @@ pub mod tests {
         assert!(ret.is_none());
     }
 
-    #[test]
-    fn test_compacting_key_range_overlap_intra_l0() {
-        // When picking L0->L0, L0's selecting_key_range should not be overlapped with L0's
-        // compacting_key_range.
-        let mut picker = create_compaction_picker_for_test();
-
-        let mut levels = Levels {
-            levels: vec![Level {
-                level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
-                table_infos: vec![generate_table(3, 1, 200, 300, 2)],
-                total_file_size: 0,
-                sub_level_id: 0,
-                uncompressed_file_size: 0,
-            }],
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![
-                generate_table(1, 1, 100, 210, 2),
-                generate_table(2, 1, 200, 250, 2),
-            ])),
-            member_table_ids: vec![1],
-            ..Default::default()
-        };
-        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-
-        let mut local_stats = LocalPickerStatistic::default();
-        let ret = picker
-            .pick_compaction(&levels, &levels_handler, &mut local_stats)
-            .unwrap();
-        ret.add_pending_task(0, &mut levels_handler);
-
-        push_table_level0_overlapping(&mut levels, generate_table(4, 1, 170, 180, 3));
-        assert!(picker
-            .pick_compaction(&levels, &levels_handler, &mut local_stats)
-            .is_none());
-    }
-
     #[test]
     fn test_skip_compact_write_amplification_limit() {
         let config: CompactionConfig = CompactionConfigBuilder::new()
@@ -821,55 +611,6 @@ pub mod tests {
         );
     }
 
-    #[test]
-    fn test_issue_11154() {
-        let mut local_stats = LocalPickerStatistic::default();
-        let mut l0 = generate_l0_overlapping_sublevels(vec![
-            vec![
-                generate_table(4, 1, 1, 200, 1),
-                generate_table(5, 1, 400, 600, 1),
-            ],
-            vec![
-                generate_table(6, 1, 1, 200, 1),
-                generate_table(7, 1, 400, 600, 1),
-            ],
-            vec![
-                generate_table(8, 1, 1, 200, 1),
-                generate_table(9, 1, 400, 600, 1),
-            ],
-            vec![generate_table(10, 1, 1, 600, 1)],
-        ]);
-        // We can set level_type only because the input above is valid.
-        for s in &mut l0.sub_levels {
-            s.level_type = LevelType::Nonoverlapping as i32;
-        }
-        let levels = Levels {
-            l0: Some(l0),
-            levels: vec![generate_level(1, vec![generate_table(3, 1, 0, 100000, 1)])],
-            member_table_ids: vec![1],
-            ..Default::default()
-        };
-        let levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-
-        // Pick with large max_compaction_bytes results all sub levels included in input.
-        let config = Arc::new(
-            CompactionConfigBuilder::new()
-                .max_compaction_bytes(800)
-                .sub_level_max_compaction_bytes(50000)
-                .max_bytes_for_level_base(500000)
-                .level0_sub_level_compact_level_count(1)
-                .build(),
-        );
-        // Only include sub-level 0 results will violate MAX_WRITE_AMPLIFICATION.
-        // So all sub-levels are included to make write amplification < MAX_WRITE_AMPLIFICATION.
-        let mut picker = LevelCompactionPicker::new(1, config);
-        let ret = picker
-            .pick_compaction(&levels, &levels_handler, &mut local_stats)
-            .unwrap();
-        // avoid add sst_10 and cause a big task
-        assert_eq!(3, ret.input_levels.len());
-    }
-
     #[test]
     fn test_l0_to_l1_break_on_pending_sub_level() {
         let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
@@ -901,6 +642,7 @@ pub mod tests {
             }],
             target_level: 1,
             target_sub_level_id: pending_level.sub_level_id,
+            ..Default::default()
         };
         assert!(!levels_handler[0].is_level_pending_compact(&pending_level));
         tier_task_input.add_pending_task(1, &mut levels_handler);
@@ -918,7 +660,6 @@ pub mod tests {
         // But stopped by pending sub-level when trying to include more sub-levels.
         let mut picker = LevelCompactionPicker::new(1, config.clone());
         let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
-
         assert!(ret.is_none());
 
         // Free the pending sub-level.
@@ -964,7 +705,6 @@ pub mod tests {
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
-        // println!("ret.input_levels: {:?}", ret.input_levels);
         // 1. trivial_move
         assert_eq!(2, ret.input_levels.len());
         assert!(ret.input_levels[1].table_infos.is_empty());
@@ -974,266 +714,7 @@ pub mod tests {
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
-        println!("ret.input_levels: {:?}", ret.input_levels);
         assert_eq!(3, ret.input_levels.len());
         assert_eq!(6, ret.input_levels[0].table_infos[0].sst_id);
     }
-
-    #[test]
-    fn test_pick_l0_intra() {
-        {
-            let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
-                vec![
-                    generate_table(6, 1, 50, 99, 1),
-                    generate_table(1, 1, 100, 200, 1),
-                    generate_table(2, 1, 250, 300, 1),
-                ],
-                vec![
-                    generate_table(3, 1, 10, 90, 1),
-                    generate_table(6, 1, 100, 110, 1),
-                ],
-                vec![
-                    generate_table(4, 1, 50, 99, 1),
-                    generate_table(5, 1, 100, 200, 1),
-                ],
-            ]);
-            let levels = Levels {
-                l0: Some(l0),
-                levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-                member_table_ids: vec![1],
-                ..Default::default()
-            };
-            let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
-            let config = Arc::new(
-                CompactionConfigBuilder::new()
-                    .level0_sub_level_compact_level_count(1)
-                    .level0_overlapping_sub_level_compact_level_count(4)
-                    .build(),
-            );
-            let mut picker = LevelCompactionPicker::new(1, config);
-            let mut local_stats = LocalPickerStatistic::default();
-            let ret = picker
-                .pick_compaction(&levels, &levels_handler, &mut local_stats)
-                .unwrap();
-            ret.add_pending_task(1, &mut levels_handler);
-            assert_eq!(
-                ret.input_levels
-                    .iter()
-                    .map(|i| i.table_infos.len())
-                    .sum::<usize>(),
-                3
-            );
-        }
-
-        {
-            // Suppose keyguard [100, 200] [300, 400]
-            // will pick sst [1, 3, 4]
-            let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
-                vec![
-                    generate_table(1, 1, 100, 200, 1),
-                    generate_table(2, 1, 300, 400, 1),
-                ],
-                vec![
-                    generate_table(3, 1, 100, 200, 1),
-                    generate_table(6, 1, 300, 500, 1),
-                ],
-                vec![
-                    generate_table(4, 1, 100, 200, 1),
-                    generate_table(5, 1, 300, 400, 1),
-                ],
-            ]);
-            let levels = Levels {
-                l0: Some(l0),
-                levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-                member_table_ids: vec![1],
-                ..Default::default()
-            };
-            let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
-            let config = Arc::new(
-                CompactionConfigBuilder::new()
-                    .level0_sub_level_compact_level_count(1)
-                    .build(),
-            );
-            let mut picker = LevelCompactionPicker::new(1, config);
-            let mut local_stats = LocalPickerStatistic::default();
-            let ret = picker
-                .pick_compaction(&levels, &levels_handler, &mut local_stats)
-                .unwrap();
-            ret.add_pending_task(1, &mut levels_handler);
-            assert_eq!(
-                ret.input_levels
-                    .iter()
-                    .map(|i| i.table_infos.len())
-                    .sum::<usize>(),
-                3
-            );
-
-            assert_eq!(4, ret.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(3, ret.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(1, ret.input_levels[2].table_infos[0].get_sst_id());
-
-            // will pick sst [2, 6, 5]
-            let ret2 = picker
-                .pick_compaction(&levels, &levels_handler, &mut local_stats)
-                .unwrap();
-
-            assert_eq!(
-                ret2.input_levels
-                    .iter()
-                    .map(|i| i.table_infos.len())
-                    .sum::<usize>(),
-                3
-            );
-
-            assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(6, ret2.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
-        }
-
-        {
-            let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
-                vec![
-                    generate_table(1, 1, 100, 149, 1),
-                    generate_table(6, 1, 150, 199, 1),
-                    generate_table(7, 1, 200, 250, 1),
-                    generate_table(2, 1, 300, 400, 1),
-                ],
-                vec![
-                    generate_table(3, 1, 100, 149, 1),
-                    generate_table(8, 1, 150, 199, 1),
-                    generate_table(9, 1, 200, 250, 1),
-                    generate_table(10, 1, 300, 400, 1),
-                ],
-                vec![
-                    generate_table(4, 1, 100, 199, 1),
-                    generate_table(11, 1, 200, 250, 1),
-                    generate_table(5, 1, 300, 350, 1),
-                ],
-            ]);
-            let levels = Levels {
-                l0: Some(l0),
-                levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-                member_table_ids: vec![1],
-                ..Default::default()
-            };
-            let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
-            let config = Arc::new(
-                CompactionConfigBuilder::new()
-                    .level0_sub_level_compact_level_count(1)
-                    .build(),
-            );
-            let mut picker = LevelCompactionPicker::new(1, config);
-            let mut local_stats = LocalPickerStatistic::default();
-            let ret = picker
-                .pick_compaction(&levels, &levels_handler, &mut local_stats)
-                .unwrap();
-            ret.add_pending_task(1, &mut levels_handler);
-            assert_eq!(
-                ret.input_levels
-                    .iter()
-                    .map(|i| i.table_infos.len())
-                    .sum::<usize>(),
-                3
-            );
-
-            assert_eq!(11, ret.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(9, ret.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(7, ret.input_levels[2].table_infos[0].get_sst_id());
-
-            let ret2 = picker
-                .pick_compaction(&levels, &levels_handler, &mut local_stats)
-                .unwrap();
-
-            assert_eq!(
-                ret2.input_levels
-                    .iter()
-                    .map(|i| i.table_infos.len())
-                    .sum::<usize>(),
-                3
-            );
-
-            assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(10, ret2.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
-        }
-    }
-
-    fn is_l0_trivial_move(compaction_input: &CompactionInput) -> bool {
-        compaction_input.input_levels.len() == 2
-            && !compaction_input.input_levels[0].table_infos.is_empty()
-            && compaction_input.input_levels[1].table_infos.is_empty()
-    }
-
-    #[test]
-    fn test_trivial_move() {
-        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-        let config = Arc::new(
-            CompactionConfigBuilder::new()
-                .level0_tier_compact_file_number(2)
-                .target_file_size_base(30)
-                .level0_sub_level_compact_level_count(20) // reject intra
-                .build(),
-        );
-        let mut picker = LevelCompactionPicker::new(1, config);
-
-        // Cannot trivial move because there is only 1 sub-level.
-        let l0 = generate_l0_overlapping_sublevels(vec![vec![
-            generate_table(1, 1, 100, 110, 1),
-            generate_table(2, 1, 150, 250, 1),
-        ]]);
-        let levels = Levels {
-            l0: Some(l0),
-            levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-            member_table_ids: vec![1],
-            ..Default::default()
-        };
-        levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
-        let mut local_stats = LocalPickerStatistic::default();
-        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
-        assert!(ret.is_none());
-
-        // Cannot trivial move because sub-levels are overlapping
-        let l0: OverlappingLevel = generate_l0_overlapping_sublevels(vec![
-            vec![
-                generate_table(1, 1, 100, 110, 1),
-                generate_table(2, 1, 150, 250, 1),
-            ],
-            vec![generate_table(3, 1, 10, 90, 1)],
-            vec![generate_table(4, 1, 10, 90, 1)],
-            vec![generate_table(5, 1, 10, 90, 1)],
-        ]);
-        let mut levels = Levels {
-            l0: Some(l0),
-            levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-            member_table_ids: vec![1],
-            ..Default::default()
-        };
-        assert!(picker
-            .pick_compaction(&levels, &levels_handler, &mut local_stats)
-            .is_none());
-
-        // Cannot trivial move because latter sub-level is overlapping
-        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32;
-        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Overlapping as i32;
-        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
-        assert!(ret.is_none());
-
-        // Cannot trivial move because former sub-level is overlapping
-        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Overlapping as i32;
-        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32;
-        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
-        assert!(ret.is_none());
-
-        // trivial move
-        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32;
-        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32;
-        let ret = picker
-            .pick_compaction(&levels, &levels_handler, &mut local_stats)
-            .unwrap();
-        assert!(is_l0_trivial_move(&ret));
-        assert_eq!(ret.input_levels[0].table_infos.len(), 1);
-    }
 }
diff --git a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
new file mode 100644
index 0000000000000..4de77467205f7
--- /dev/null
+++ b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
@@ -0,0 +1,208 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use risingwave_pb::hummock::CompactionConfig;
+
+use super::{CompactionInput, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT};
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum ValidationRuleType {
+    Tier = 0,
+    Intra = 1,
+    ToBase = 2,
+}
+
+pub struct CompactionTaskValidator {
+    validation_rules: HashMap<ValidationRuleType, Box<dyn CompactionTaskValidationRule>>,
+}
+
+impl CompactionTaskValidator {
+    pub fn new(config: Arc<CompactionConfig>) -> Self {
+        let mut validation_rules: HashMap<
+            ValidationRuleType,
+            Box<dyn CompactionTaskValidationRule>,
+        > = HashMap::default();
+
+        validation_rules.insert(
+            ValidationRuleType::Tier,
+            Box::new(TierCompactionTaskValidationRule {
+                config: config.clone(),
+                enable: true,
+            }),
+        );
+
+        validation_rules.insert(
+            ValidationRuleType::Intra,
+            Box::new(IntraCompactionTaskValidationRule {
+                config: config.clone(),
+                enable: true,
+            }),
+        );
+
+        validation_rules.insert(
+            ValidationRuleType::ToBase,
+            Box::new(BaseCompactionTaskValidationRule {
+                config,
+                enable: true,
+            }),
+        );
+
+        CompactionTaskValidator { validation_rules }
+    }
+
+    pub fn valid_compact_task(
+        &self,
+        input: &CompactionInput,
+        picker_type: ValidationRuleType,
+        stats: &mut LocalPickerStatistic,
+    ) -> bool {
+        self.validation_rules
+            .get(&picker_type)
+            .unwrap()
+            .validate(input, stats)
+    }
+}
+
+pub trait CompactionTaskValidationRule {
+    fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool;
+}
+
+struct TierCompactionTaskValidationRule {
+    config: Arc<CompactionConfig>,
+    enable: bool,
+}
+
+impl CompactionTaskValidationRule for TierCompactionTaskValidationRule {
+    fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
+        if !self.enable {
+            return true;
+        }
+
+        // so the design here wants to merge multiple overlapping-levels in one compaction
+        let max_compaction_bytes = std::cmp::min(
+            self.config.max_compaction_bytes,
+            self.config.sub_level_max_compaction_bytes
+                * self.config.level0_overlapping_sub_level_compact_level_count as u64,
+        );
+
+        // Limit sstable file count to avoid using too much memory.
+        let overlapping_max_compact_file_numer = std::cmp::min(
+            self.config.level0_max_compact_file_number,
+            MAX_COMPACT_LEVEL_COUNT as u64,
+        );
+
+        let waiting_enough_files = {
+            if input.select_input_size > max_compaction_bytes {
+                false
+            } else {
+                input.total_file_count <= overlapping_max_compact_file_numer
+            }
+        };
+
+        // If waiting_enough_files is not satisfied, we will raise the priority of the number of
+        // levels to ensure that we can merge as many sub_levels as possible
+        let tier_sub_level_compact_level_count =
+            self.config.level0_overlapping_sub_level_compact_level_count as usize;
+        if input.input_levels.len() < tier_sub_level_compact_level_count && waiting_enough_files {
+            stats.skip_by_count_limit += 1;
+            return false;
+        }
+
+        true
+    }
+}
+
+struct IntraCompactionTaskValidationRule {
+    config: Arc<CompactionConfig>,
+    enable: bool,
+}
+
+impl CompactionTaskValidationRule for IntraCompactionTaskValidationRule {
+    fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
+        if !self.enable {
+            return true;
+        }
+
+        let intra_sub_level_compact_level_count =
+            self.config.level0_sub_level_compact_level_count as usize;
+
+        if input.input_levels.len() < intra_sub_level_compact_level_count {
+            return false;
+        }
+
+        let mut max_level_size = 0;
+        for select_level in &input.input_levels {
+            let level_select_size = select_level
+                .table_infos
+                .iter()
+                .map(|sst| sst.file_size)
+                .sum::<u64>();
+
+            max_level_size = std::cmp::max(max_level_size, level_select_size);
+        }
+
+        // This limitation would keep our write-amplification no more than
+        // ln(max_compaction_bytes/flush_level_bytes) /
+        // ln(self.config.level0_sub_level_compact_level_count/2) Here we only use half
+        // of level0_sub_level_compact_level_count just for convenient.
+        let is_write_amp_large =
+            max_level_size * self.config.level0_sub_level_compact_level_count as u64 / 2
+                >= input.select_input_size;
+
+        if is_write_amp_large && input.total_file_count < self.config.level0_max_compact_file_number
+        {
+            stats.skip_by_write_amp_limit += 1;
+            return false;
+        }
+
+        if input.input_levels.len() < intra_sub_level_compact_level_count
+            && input.total_file_count < self.config.level0_max_compact_file_number
+        {
+            stats.skip_by_count_limit += 1;
+            return false;
+        }
+
+        true
+    }
+}
+
+struct BaseCompactionTaskValidationRule {
+    config: Arc<CompactionConfig>,
+    enable: bool,
+}
+
+impl CompactionTaskValidationRule for BaseCompactionTaskValidationRule {
+    fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
+        if !self.enable {
+            return true;
+        }
+
+        // The size of target level may be too large, we shall skip this compact task and wait
+        //  the data in base level compact to lower level.
+        if input.target_input_size > self.config.max_compaction_bytes {
+            stats.skip_by_count_limit += 1;
+            return false;
+        }
+
+        if input.select_input_size < input.target_input_size {
+            stats.skip_by_write_amp_limit += 1;
+            return false;
+        }
+
+        true
+    }
+}
diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
new file mode 100644
index 0000000000000..541b93254172b
--- /dev/null
+++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
@@ -0,0 +1,675 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use risingwave_pb::hummock::hummock_version::Levels;
+use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel};
+
+use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker;
+use super::{
+    CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic,
+    ValidationRuleType,
+};
+use crate::hummock::compaction::create_overlap_strategy;
+use crate::hummock::compaction::picker::TrivialMovePicker;
+use crate::hummock::level_handler::LevelHandler;
+
+pub struct IntraCompactionPicker {
+    config: Arc<CompactionConfig>,
+    compaction_task_validator: Arc<CompactionTaskValidator>,
+}
+
+impl CompactionPicker for IntraCompactionPicker {
+    fn pick_compaction(
+        &mut self,
+        levels: &Levels,
+        level_handlers: &[LevelHandler],
+        stats: &mut LocalPickerStatistic,
+    ) -> Option<CompactionInput> {
+        let l0 = levels.l0.as_ref().unwrap();
+        if l0.sub_levels.is_empty() {
+            return None;
+        }
+        if l0.sub_levels[0].level_type != LevelType::Nonoverlapping as i32
+            && l0.sub_levels[0].table_infos.len() > 1
+        {
+            stats.skip_by_overlapping += 1;
+            return None;
+        }
+
+        let is_l0_pending_compact =
+            level_handlers[0].is_level_all_pending_compact(&l0.sub_levels[0]);
+
+        if is_l0_pending_compact {
+            stats.skip_by_pending_files += 1;
+            return None;
+        }
+
+        if let Some(ret) = self.pick_l0_intra(l0, &level_handlers[0], stats) {
+            return Some(ret);
+        }
+
+        self.pick_l0_trivial_move_file(l0, level_handlers, stats)
+    }
+}
+
+impl IntraCompactionPicker {
+    #[cfg(test)]
+    pub fn new(config: Arc<CompactionConfig>) -> IntraCompactionPicker {
+        IntraCompactionPicker {
+            compaction_task_validator: Arc::new(CompactionTaskValidator::new(config.clone())),
+            config,
+        }
+    }
+
+    pub fn new_with_validator(
+        config: Arc<CompactionConfig>,
+        compaction_task_validator: Arc<CompactionTaskValidator>,
+    ) -> IntraCompactionPicker {
+        IntraCompactionPicker {
+            config,
+            compaction_task_validator,
+        }
+    }
+
+    fn pick_l0_intra(
+        &self,
+        l0: &OverlappingLevel,
+        level_handler: &LevelHandler,
+        stats: &mut LocalPickerStatistic,
+    ) -> Option<CompactionInput> {
+        let overlap_strategy = create_overlap_strategy(self.config.compaction_mode());
+
+        for (idx, level) in l0.sub_levels.iter().enumerate() {
+            if level.level_type() != LevelType::Nonoverlapping
+                || level.total_file_size > self.config.sub_level_max_compaction_bytes
+            {
+                continue;
+            }
+
+            if level_handler.is_level_all_pending_compact(level) {
+                continue;
+            }
+
+            let max_compaction_bytes = std::cmp::min(
+                self.config.max_compaction_bytes,
+                self.config.sub_level_max_compaction_bytes,
+            );
+
+            let non_overlap_sub_level_picker = NonOverlapSubLevelPicker::new(
+                self.config.sub_level_max_compaction_bytes / 2,
+                max_compaction_bytes,
+                self.config.level0_sub_level_compact_level_count as usize,
+                self.config.level0_max_compact_file_number,
+                overlap_strategy.clone(),
+            );
+
+            let l0_select_tables_vec = non_overlap_sub_level_picker
+                .pick_l0_multi_non_overlap_level(&l0.sub_levels[idx..], level_handler);
+
+            if l0_select_tables_vec.is_empty() {
+                continue;
+            }
+
+            let mut select_input_size = 0;
+            let mut total_file_count = 0;
+            for input in l0_select_tables_vec {
+                let mut max_level_size = 0;
+                for level_select_table in &input.sstable_infos {
+                    let level_select_size = level_select_table
+                        .iter()
+                        .map(|sst| sst.file_size)
+                        .sum::<u64>();
+
+                    max_level_size = std::cmp::max(max_level_size, level_select_size);
+                }
+
+                let mut select_level_inputs = Vec::with_capacity(input.sstable_infos.len());
+                for level_select_sst in input.sstable_infos {
+                    if level_select_sst.is_empty() {
+                        continue;
+                    }
+                    select_level_inputs.push(InputLevel {
+                        level_idx: 0,
+                        level_type: LevelType::Nonoverlapping as i32,
+                        table_infos: level_select_sst,
+                    });
+
+                    select_input_size += input.total_file_size;
+                    total_file_count += input.total_file_count;
+                }
+                select_level_inputs.reverse();
+
+                let result = CompactionInput {
+                    input_levels: select_level_inputs,
+                    target_sub_level_id: level.sub_level_id,
+                    select_input_size,
+                    total_file_count: total_file_count as u64,
+                    ..Default::default()
+                };
+
+                if !self.compaction_task_validator.valid_compact_task(
+                    &result,
+                    ValidationRuleType::Intra,
+                    stats,
+                ) {
+                    continue;
+                }
+
+                return Some(result);
+            }
+        }
+
+        None
+    }
+
+    fn pick_l0_trivial_move_file(
+        &self,
+        l0: &OverlappingLevel,
+        level_handlers: &[LevelHandler],
+        stats: &mut LocalPickerStatistic,
+    ) -> Option<CompactionInput> {
+        let overlap_strategy = create_overlap_strategy(self.config.compaction_mode());
+
+        for (idx, level) in l0.sub_levels.iter().enumerate() {
+            if level.level_type == LevelType::Overlapping as i32 || idx + 1 >= l0.sub_levels.len() {
+                continue;
+            }
+
+            if l0.sub_levels[idx + 1].level_type == LevelType::Overlapping as i32 {
+                continue;
+            }
+
+            let trivial_move_picker = TrivialMovePicker::new(0, 0, overlap_strategy.clone());
+
+            let select_sst = trivial_move_picker.pick_trivial_move_sst(
+                &l0.sub_levels[idx + 1].table_infos,
+                &level.table_infos,
+                level_handlers,
+                stats,
+            );
+
+            // only pick tables for trivial move
+            if select_sst.is_none() {
+                continue;
+            }
+
+            let select_sst = select_sst.unwrap();
+
+            // support trivial move cross multi sub_levels
+            let mut overlap = overlap_strategy.create_overlap_info();
+            overlap.update(&select_sst);
+
+            assert!(overlap
+                .check_multiple_overlap(&l0.sub_levels[idx].table_infos)
+                .is_empty());
+            let mut target_level_idx = idx;
+            while target_level_idx > 0 {
+                if l0.sub_levels[target_level_idx - 1].level_type
+                    != LevelType::Nonoverlapping as i32
+                    || !overlap
+                        .check_multiple_overlap(&l0.sub_levels[target_level_idx - 1].table_infos)
+                        .is_empty()
+                {
+                    break;
+                }
+                target_level_idx -= 1;
+            }
+
+            let select_input_size = select_sst.file_size;
+            let input_levels = vec![
+                InputLevel {
+                    level_idx: 0,
+                    level_type: LevelType::Nonoverlapping as i32,
+                    table_infos: vec![select_sst],
+                },
+                InputLevel {
+                    level_idx: 0,
+                    level_type: LevelType::Nonoverlapping as i32,
+                    table_infos: vec![],
+                },
+            ];
+            return Some(CompactionInput {
+                input_levels,
+                target_level: 0,
+                target_sub_level_id: l0.sub_levels[target_level_idx].sub_level_id,
+                select_input_size,
+                total_file_count: 1,
+                ..Default::default()
+            });
+        }
+        None
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use risingwave_pb::hummock::Level;
+
+    use super::*;
+    use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
+    use crate::hummock::compaction::level_selector::tests::{
+        generate_l0_nonoverlapping_multi_sublevels, generate_l0_nonoverlapping_sublevels,
+        generate_l0_overlapping_sublevels, generate_level, generate_table,
+        push_table_level0_overlapping, push_tables_level0_nonoverlapping,
+    };
+    use crate::hummock::compaction::TierCompactionPicker;
+
+    fn create_compaction_picker_for_test() -> IntraCompactionPicker {
+        let config = Arc::new(
+            CompactionConfigBuilder::new()
+                .level0_tier_compact_file_number(2)
+                .level0_sub_level_compact_level_count(1)
+                .build(),
+        );
+        IntraCompactionPicker::new(config)
+    }
+
+    #[test]
+    fn test_l0_to_l1_compact_conflict() {
+        // When picking L0->L1, L0's selecting_key_range should not be overlapped with L0's
+        // compacting_key_range.
+        let mut picker = create_compaction_picker_for_test();
+        let levels = vec![Level {
+            level_idx: 1,
+            level_type: LevelType::Nonoverlapping as i32,
+            table_infos: vec![],
+            total_file_size: 0,
+            sub_level_id: 0,
+            uncompressed_file_size: 0,
+        }];
+        let mut levels = Levels {
+            levels,
+            l0: Some(OverlappingLevel {
+                sub_levels: vec![],
+                total_file_size: 0,
+                uncompressed_file_size: 0,
+            }),
+            member_table_ids: vec![1],
+            ..Default::default()
+        };
+        push_tables_level0_nonoverlapping(
+            &mut levels,
+            vec![
+                generate_table(1, 1, 100, 300, 2),
+                generate_table(2, 1, 350, 500, 2),
+            ],
+        );
+        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+
+        let mut local_stats = LocalPickerStatistic::default();
+        let ret = picker
+            .pick_compaction(&levels, &levels_handler, &mut local_stats)
+            .unwrap();
+        // trivial_move
+        ret.add_pending_task(0, &mut levels_handler); // pending only for test
+        push_tables_level0_nonoverlapping(&mut levels, vec![generate_table(3, 1, 250, 300, 3)]);
+        let config: CompactionConfig = CompactionConfigBuilder::new()
+            .level0_tier_compact_file_number(2)
+            .max_compaction_bytes(1000)
+            .sub_level_max_compaction_bytes(150)
+            .max_bytes_for_level_multiplier(1)
+            .level0_sub_level_compact_level_count(3)
+            .build();
+        let mut picker = TierCompactionPicker::new(Arc::new(config));
+
+        let ret: Option<CompactionInput> =
+            picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
+        assert!(ret.is_none());
+    }
+
+    #[test]
+    fn test_compacting_key_range_overlap_intra_l0() {
+        // When picking L0->L0, L0's selecting_key_range should not be overlapped with L0's
+        // compacting_key_range.
+        let mut picker = create_compaction_picker_for_test();
+
+        let mut levels = Levels {
+            levels: vec![Level {
+                level_idx: 1,
+                level_type: LevelType::Nonoverlapping as i32,
+                table_infos: vec![generate_table(3, 1, 200, 300, 2)],
+                total_file_size: 0,
+                sub_level_id: 0,
+                uncompressed_file_size: 0,
+            }],
+            l0: Some(generate_l0_nonoverlapping_sublevels(vec![
+                generate_table(1, 1, 100, 210, 2),
+                generate_table(2, 1, 200, 250, 2),
+            ])),
+            member_table_ids: vec![1],
+            ..Default::default()
+        };
+        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+
+        let mut local_stats = LocalPickerStatistic::default();
+        let ret = picker
+            .pick_compaction(&levels, &levels_handler, &mut local_stats)
+            .unwrap();
+        ret.add_pending_task(0, &mut levels_handler);
+
+        push_table_level0_overlapping(&mut levels, generate_table(4, 1, 170, 180, 3));
+        assert!(picker
+            .pick_compaction(&levels, &levels_handler, &mut local_stats)
+            .is_none());
+    }
+
+    #[test]
+    fn test_pick_l0_intra() {
+        {
+            let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
+                vec![
+                    generate_table(6, 1, 50, 99, 1),
+                    generate_table(1, 1, 100, 200, 1),
+                    generate_table(2, 1, 250, 300, 1),
+                ],
+                vec![
+                    generate_table(3, 1, 10, 90, 1),
+                    generate_table(6, 1, 100, 110, 1),
+                ],
+                vec![
+                    generate_table(4, 1, 50, 99, 1),
+                    generate_table(5, 1, 100, 200, 1),
+                ],
+            ]);
+            let levels = Levels {
+                l0: Some(l0),
+                levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
+                member_table_ids: vec![1],
+                ..Default::default()
+            };
+            let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+            let config = Arc::new(
+                CompactionConfigBuilder::new()
+                    .level0_sub_level_compact_level_count(1)
+                    .level0_overlapping_sub_level_compact_level_count(4)
+                    .build(),
+            );
+            let mut picker = IntraCompactionPicker::new(config);
+            let mut local_stats = LocalPickerStatistic::default();
+            let ret = picker
+                .pick_compaction(&levels, &levels_handler, &mut local_stats)
+                .unwrap();
+            ret.add_pending_task(1, &mut levels_handler);
+            assert_eq!(
+                ret.input_levels
+                    .iter()
+                    .map(|i| i.table_infos.len())
+                    .sum::<usize>(),
+                3
+            );
+        }
+
+        {
+            // Suppose keyguard [100, 200] [300, 400]
+            // will pick sst [1, 3, 4]
+            let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
+                vec![
+                    generate_table(1, 1, 100, 200, 1),
+                    generate_table(2, 1, 300, 400, 1),
+                ],
+                vec![
+                    generate_table(3, 1, 100, 200, 1),
+                    generate_table(6, 1, 300, 500, 1),
+                ],
+                vec![
+                    generate_table(4, 1, 100, 200, 1),
+                    generate_table(5, 1, 300, 400, 1),
+                ],
+            ]);
+            let levels = Levels {
+                l0: Some(l0),
+                levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
+                member_table_ids: vec![1],
+                ..Default::default()
+            };
+            let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+            let config = Arc::new(
+                CompactionConfigBuilder::new()
+                    .level0_sub_level_compact_level_count(1)
+                    .build(),
+            );
+            let mut picker = IntraCompactionPicker::new(config);
+            let mut local_stats = LocalPickerStatistic::default();
+            let ret = picker
+                .pick_compaction(&levels, &levels_handler, &mut local_stats)
+                .unwrap();
+            ret.add_pending_task(1, &mut levels_handler);
+            assert_eq!(
+                ret.input_levels
+                    .iter()
+                    .map(|i| i.table_infos.len())
+                    .sum::<usize>(),
+                3
+            );
+
+            assert_eq!(4, ret.input_levels[0].table_infos[0].get_sst_id());
+            assert_eq!(3, ret.input_levels[1].table_infos[0].get_sst_id());
+            assert_eq!(1, ret.input_levels[2].table_infos[0].get_sst_id());
+
+            // will pick sst [2, 6, 5]
+            let ret2 = picker
+                .pick_compaction(&levels, &levels_handler, &mut local_stats)
+                .unwrap();
+
+            assert_eq!(
+                ret2.input_levels
+                    .iter()
+                    .map(|i| i.table_infos.len())
+                    .sum::<usize>(),
+                3
+            );
+
+            assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
+            assert_eq!(6, ret2.input_levels[1].table_infos[0].get_sst_id());
+            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
+        }
+
+        {
+            let l0 = generate_l0_nonoverlapping_multi_sublevels(vec![
+                vec![
+                    generate_table(1, 1, 100, 149, 1),
+                    generate_table(6, 1, 150, 199, 1),
+                    generate_table(7, 1, 200, 250, 1),
+                    generate_table(2, 1, 300, 400, 1),
+                ],
+                vec![
+                    generate_table(3, 1, 100, 149, 1),
+                    generate_table(8, 1, 150, 199, 1),
+                    generate_table(9, 1, 200, 250, 1),
+                    generate_table(10, 1, 300, 400, 1),
+                ],
+                vec![
+                    generate_table(4, 1, 100, 199, 1),
+                    generate_table(11, 1, 200, 250, 1),
+                    generate_table(5, 1, 300, 350, 1),
+                ],
+            ]);
+            let levels = Levels {
+                l0: Some(l0),
+                levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
+                member_table_ids: vec![1],
+                ..Default::default()
+            };
+            let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+            let config = Arc::new(
+                CompactionConfigBuilder::new()
+                    .level0_sub_level_compact_level_count(1)
+                    .build(),
+            );
+            let mut picker = IntraCompactionPicker::new(config);
+            let mut local_stats = LocalPickerStatistic::default();
+            let ret = picker
+                .pick_compaction(&levels, &levels_handler, &mut local_stats)
+                .unwrap();
+            ret.add_pending_task(1, &mut levels_handler);
+            assert_eq!(
+                ret.input_levels
+                    .iter()
+                    .map(|i| i.table_infos.len())
+                    .sum::<usize>(),
+                3
+            );
+
+            assert_eq!(11, ret.input_levels[0].table_infos[0].get_sst_id());
+            assert_eq!(9, ret.input_levels[1].table_infos[0].get_sst_id());
+            assert_eq!(7, ret.input_levels[2].table_infos[0].get_sst_id());
+
+            let ret2 = picker
+                .pick_compaction(&levels, &levels_handler, &mut local_stats)
+                .unwrap();
+
+            assert_eq!(
+                ret2.input_levels
+                    .iter()
+                    .map(|i| i.table_infos.len())
+                    .sum::<usize>(),
+                3
+            );
+
+            assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
+            assert_eq!(10, ret2.input_levels[1].table_infos[0].get_sst_id());
+            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
+        }
+    }
+
+    fn is_l0_trivial_move(compaction_input: &CompactionInput) -> bool {
+        compaction_input.input_levels.len() == 2
+            && !compaction_input.input_levels[0].table_infos.is_empty()
+            && compaction_input.input_levels[1].table_infos.is_empty()
+    }
+
+    #[test]
+    fn test_trivial_move() {
+        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+        let config = Arc::new(
+            CompactionConfigBuilder::new()
+                .level0_tier_compact_file_number(2)
+                .target_file_size_base(30)
+                .level0_sub_level_compact_level_count(20) // reject intra
+                .build(),
+        );
+        let mut picker = IntraCompactionPicker::new(config);
+
+        // Cannot trivial move because there is only 1 sub-level.
+        let l0 = generate_l0_overlapping_sublevels(vec![vec![
+            generate_table(1, 1, 100, 110, 1),
+            generate_table(2, 1, 150, 250, 1),
+        ]]);
+        let levels = Levels {
+            l0: Some(l0),
+            levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
+            member_table_ids: vec![1],
+            ..Default::default()
+        };
+        levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+        let mut local_stats = LocalPickerStatistic::default();
+        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
+        assert!(ret.is_none());
+
+        // Cannot trivial move because sub-levels are overlapping
+        let l0: OverlappingLevel = generate_l0_overlapping_sublevels(vec![
+            vec![
+                generate_table(1, 1, 100, 110, 1),
+                generate_table(2, 1, 150, 250, 1),
+            ],
+            vec![generate_table(3, 1, 10, 90, 1)],
+            vec![generate_table(4, 1, 10, 90, 1)],
+            vec![generate_table(5, 1, 10, 90, 1)],
+        ]);
+        let mut levels = Levels {
+            l0: Some(l0),
+            levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
+            member_table_ids: vec![1],
+            ..Default::default()
+        };
+        assert!(picker
+            .pick_compaction(&levels, &levels_handler, &mut local_stats)
+            .is_none());
+
+        // Cannot trivial move because latter sub-level is overlapping
+        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32;
+        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Overlapping as i32;
+        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
+        assert!(ret.is_none());
+
+        // Cannot trivial move because former sub-level is overlapping
+        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Overlapping as i32;
+        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32;
+        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
+        assert!(ret.is_none());
+
+        // trivial move
+        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32;
+        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32;
+        let ret = picker
+            .pick_compaction(&levels, &levels_handler, &mut local_stats)
+            .unwrap();
+        assert!(is_l0_trivial_move(&ret));
+        assert_eq!(ret.input_levels[0].table_infos.len(), 1);
+    }
+
+    #[test]
+    fn test_issue_11154() {
+        let mut local_stats = LocalPickerStatistic::default();
+        let mut l0 = generate_l0_overlapping_sublevels(vec![
+            vec![
+                generate_table(4, 1, 1, 200, 1),
+                generate_table(5, 1, 400, 600, 1),
+            ],
+            vec![
+                generate_table(6, 1, 1, 200, 1),
+                generate_table(7, 1, 400, 600, 1),
+            ],
+            vec![
+                generate_table(8, 1, 1, 200, 1),
+                generate_table(9, 1, 400, 600, 1),
+            ],
+            vec![generate_table(10, 1, 1, 600, 1)],
+        ]);
+        // We can set level_type only because the input above is valid.
+        for s in &mut l0.sub_levels {
+            s.level_type = LevelType::Nonoverlapping as i32;
+        }
+        let levels = Levels {
+            l0: Some(l0),
+            levels: vec![generate_level(1, vec![generate_table(3, 1, 0, 100000, 1)])],
+            member_table_ids: vec![1],
+            ..Default::default()
+        };
+        let levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+
+        // Pick with large max_compaction_bytes results all sub levels included in input.
+        let config = Arc::new(
+            CompactionConfigBuilder::new()
+                .max_compaction_bytes(800)
+                .sub_level_max_compaction_bytes(50000)
+                .max_bytes_for_level_base(500000)
+                .level0_sub_level_compact_level_count(1)
+                .build(),
+        );
+        // Only include sub-level 0 results will violate MAX_WRITE_AMPLIFICATION.
+        // So all sub-levels are included to make write amplification < MAX_WRITE_AMPLIFICATION.
+        let mut picker = IntraCompactionPicker::new(config);
+        let ret = picker
+            .pick_compaction(&levels, &levels_handler, &mut local_stats)
+            .unwrap();
+        // avoid add sst_10 and cause a big task
+        assert_eq!(3, ret.input_levels.len());
+    }
+}
diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
index a6942b2e4d680..e8f8c908d0fd3 100644
--- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
@@ -101,6 +101,7 @@ impl ManualCompactionPicker {
             input_levels,
             target_level: 0,
             target_sub_level_id: sub_level_id,
+            ..Default::default()
         })
     }
 
@@ -170,6 +171,7 @@ impl ManualCompactionPicker {
             input_levels,
             target_level: self.target_level,
             target_sub_level_id: 0,
+            ..Default::default()
         })
     }
 
@@ -301,6 +303,9 @@ impl CompactionPicker for ManualCompactionPicker {
         }
 
         Some(CompactionInput {
+            select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+            target_input_size: target_input_ssts.iter().map(|sst| sst.file_size).sum(),
+            total_file_count: (select_input_ssts.len() + target_input_ssts.len()) as u64,
             input_levels: vec![
                 InputLevel {
                     level_idx: level as u32,
@@ -314,7 +319,7 @@ impl CompactionPicker for ManualCompactionPicker {
                 },
             ],
             target_level,
-            target_sub_level_id: 0,
+            ..Default::default()
         })
     }
 }
diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
index fc8430a536ae3..0cf44795e0acb 100644
--- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
@@ -20,10 +20,9 @@ use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
 use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::{InputLevel, Level, LevelType, SstableInfo};
 
-use super::{CompactionInput, CompactionPicker, LocalPickerStatistic};
+use super::{CompactionInput, CompactionPicker, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT};
 use crate::hummock::compaction::overlap_strategy::OverlapStrategy;
 use crate::hummock::level_handler::LevelHandler;
-pub const MAX_LEVEL_COUNT: usize = 42;
 
 pub struct MinOverlappingPicker {
     level: usize,
@@ -130,6 +129,9 @@ impl CompactionPicker for MinOverlappingPicker {
             return None;
         }
         Some(CompactionInput {
+            select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+            target_input_size: target_input_ssts.iter().map(|sst| sst.file_size).sum(),
+            total_file_count: (select_input_ssts.len() + target_input_ssts.len()) as u64,
             input_levels: vec![
                 InputLevel {
                     level_idx: self.level as u32,
@@ -143,7 +145,7 @@ impl CompactionPicker for MinOverlappingPicker {
                 },
             ],
             target_level: self.target_level,
-            target_sub_level_id: 0,
+            ..Default::default()
         })
     }
 }
@@ -300,7 +302,7 @@ impl NonOverlapSubLevelPicker {
                 .iter()
                 .filter(|ssts| !ssts.is_empty())
                 .count()
-                > MAX_LEVEL_COUNT
+                > MAX_COMPACT_LEVEL_COUNT
             {
                 break;
             }
diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs
index 86f3736288be6..15e7a61f548ee 100644
--- a/src/meta/src/hummock/compaction/picker/mod.rs
+++ b/src/meta/src/hummock/compaction/picker/mod.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod base_level_compaction_picker;
+mod intra_compaction_picker;
 mod manual_compaction_picker;
 mod min_overlap_compaction_picker;
 mod space_reclaim_compaction_picker;
@@ -21,7 +22,11 @@ mod tombstone_reclaim_compaction_picker;
 mod trivial_move_compaction_picker;
 mod ttl_reclaim_compaction_picker;
 
+mod compaction_task_validator;
+
 pub use base_level_compaction_picker::LevelCompactionPicker;
+pub use compaction_task_validator::{CompactionTaskValidator, ValidationRuleType};
+pub use intra_compaction_picker::IntraCompactionPicker;
 pub use manual_compaction_picker::ManualCompactionPicker;
 pub use min_overlap_compaction_picker::MinOverlappingPicker;
 use risingwave_pb::hummock::hummock_version::Levels;
@@ -36,6 +41,8 @@ pub use ttl_reclaim_compaction_picker::{TtlPickerState, TtlReclaimCompactionPick
 
 use crate::hummock::level_handler::LevelHandler;
 
+pub const MAX_COMPACT_LEVEL_COUNT: usize = 42;
+
 #[derive(Default)]
 pub struct LocalPickerStatistic {
     pub skip_by_write_amp_limit: u64,
@@ -43,10 +50,15 @@ pub struct LocalPickerStatistic {
     pub skip_by_pending_files: u64,
     pub skip_by_overlapping: u64,
 }
+
+#[derive(Default)]
 pub struct CompactionInput {
     pub input_levels: Vec<InputLevel>,
     pub target_level: usize,
     pub target_sub_level_id: u64,
+    pub select_input_size: u64,
+    pub target_input_size: u64,
+    pub total_file_count: u64,
 }
 
 impl CompactionInput {
diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
index 5a1360c87c015..a3ff21831fef8 100644
--- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
@@ -80,6 +80,8 @@ impl SpaceReclaimCompactionPicker {
                 }
                 if !select_input_ssts.is_empty() {
                     return Some(CompactionInput {
+                        select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+                        total_file_count: select_input_ssts.len() as u64,
                         input_levels: vec![
                             InputLevel {
                                 level_idx: level.level_idx,
@@ -94,6 +96,7 @@ impl SpaceReclaimCompactionPicker {
                         ],
                         target_level: level.level_idx as usize,
                         target_sub_level_id: level.sub_level_id,
+                        ..Default::default()
                     });
                 }
             }
@@ -135,6 +138,8 @@ impl SpaceReclaimCompactionPicker {
             // turn to next_round
             if !select_input_ssts.is_empty() {
                 return Some(CompactionInput {
+                    select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+                    total_file_count: select_input_ssts.len() as u64,
                     input_levels: vec![
                         InputLevel {
                             level_idx: state.last_level as u32,
@@ -148,7 +153,7 @@ impl SpaceReclaimCompactionPicker {
                         },
                     ],
                     target_level: state.last_level,
-                    target_sub_level_id: 0,
+                    ..Default::default()
                 });
             }
             state.last_level += 1;
diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
index 6ebb2fed50364..99b17694f528e 100644
--- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
@@ -19,17 +19,35 @@ use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
 use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel};
 
-use super::{CompactionInput, CompactionPicker, LocalPickerStatistic};
-use crate::hummock::compaction::picker::min_overlap_compaction_picker::MAX_LEVEL_COUNT;
+use super::{
+    CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic,
+    ValidationRuleType,
+};
+use crate::hummock::compaction::picker::MAX_COMPACT_LEVEL_COUNT;
 use crate::hummock::level_handler::LevelHandler;
 
 pub struct TierCompactionPicker {
     config: Arc<CompactionConfig>,
+    compaction_task_validator: Arc<CompactionTaskValidator>,
 }
 
 impl TierCompactionPicker {
+    #[cfg(test)]
     pub fn new(config: Arc<CompactionConfig>) -> TierCompactionPicker {
-        TierCompactionPicker { config }
+        TierCompactionPicker {
+            compaction_task_validator: Arc::new(CompactionTaskValidator::new(config.clone())),
+            config,
+        }
+    }
+
+    pub fn new_with_validator(
+        config: Arc<CompactionConfig>,
+        compaction_task_validator: Arc<CompactionTaskValidator>,
+    ) -> TierCompactionPicker {
+        TierCompactionPicker {
+            config,
+            compaction_task_validator,
+        }
     }
 
     fn pick_overlapping_level(
@@ -66,9 +84,16 @@ impl TierCompactionPicker {
 
             if can_concat(&input_level.table_infos) {
                 return Some(CompactionInput {
+                    select_input_size: input_level
+                        .table_infos
+                        .iter()
+                        .map(|sst| sst.file_size)
+                        .sum(),
+                    total_file_count: input_level.table_infos.len() as u64,
                     input_levels: vec![input_level],
                     target_level: 0,
                     target_sub_level_id: level.sub_level_id,
+                    ..Default::default()
                 });
             }
 
@@ -87,29 +112,15 @@ impl TierCompactionPicker {
             // Limit sstable file count to avoid using too much memory.
             let overlapping_max_compact_file_numer = std::cmp::min(
                 self.config.level0_max_compact_file_number,
-                MAX_LEVEL_COUNT as u64,
+                MAX_COMPACT_LEVEL_COUNT as u64,
             );
-            let mut waiting_enough_files = {
-                if compaction_bytes > max_compaction_bytes {
-                    false
-                } else {
-                    compact_file_count <= overlapping_max_compact_file_numer
-                }
-            };
 
             for other in &l0.sub_levels[idx + 1..] {
                 if compaction_bytes > max_compaction_bytes {
-                    waiting_enough_files = false;
                     break;
                 }
 
                 if compact_file_count > overlapping_max_compact_file_numer {
-                    waiting_enough_files = false;
-                    break;
-                }
-
-                if other.level_type() != LevelType::Overlapping {
-                    waiting_enough_files = false;
                     break;
                 }
 
@@ -126,24 +137,26 @@ impl TierCompactionPicker {
                 });
             }
 
-            // If waiting_enough_files is not satisfied, we will raise the priority of the number of
-            // levels to ensure that we can merge as many sub_levels as possible
-            let tier_sub_level_compact_level_count =
-                self.config.level0_overlapping_sub_level_compact_level_count as usize;
-            if select_level_inputs.len() < tier_sub_level_compact_level_count
-                && waiting_enough_files
-            {
-                stats.skip_by_count_limit += 1;
-                continue;
-            }
-
             select_level_inputs.reverse();
 
-            return Some(CompactionInput {
+            let result = CompactionInput {
                 input_levels: select_level_inputs,
                 target_level: 0,
                 target_sub_level_id: level.sub_level_id,
-            });
+                select_input_size: compaction_bytes,
+                target_input_size: 0,
+                total_file_count: compact_file_count,
+            };
+
+            if !self.compaction_task_validator.valid_compact_task(
+                &result,
+                ValidationRuleType::Tier,
+                stats,
+            ) {
+                continue;
+            }
+
+            return Some(result);
         }
         None
     }
diff --git a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
index 97d8fa995d8b1..994bfbc5ea557 100644
--- a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
@@ -117,6 +117,14 @@ impl TombstoneReclaimCompactionPicker {
                     }
                 };
                 return Some(CompactionInput {
+                    select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+                    target_input_size: target_level
+                        .table_infos
+                        .iter()
+                        .map(|sst| sst.file_size)
+                        .sum(),
+                    total_file_count: (select_input_ssts.len() + target_level.table_infos.len())
+                        as u64,
                     target_level: target_level.level_idx as usize,
                     input_levels: vec![
                         InputLevel {
@@ -126,7 +134,7 @@ impl TombstoneReclaimCompactionPicker {
                         },
                         target_level,
                     ],
-                    target_sub_level_id: 0,
+                    ..Default::default()
                 });
             }
             state.last_level += 1;
diff --git a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs
index 89f794e04efb2..4bfbca0c5fb59 100644
--- a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs
@@ -79,6 +79,8 @@ impl TrivialMovePicker {
             self.pick_trivial_move_sst(select_tables, target_tables, level_handlers, stats)
         {
             return Some(CompactionInput {
+                select_input_size: trivial_move_sst.file_size,
+                total_file_count: 1,
                 input_levels: vec![
                     InputLevel {
                         level_idx: self.level as u32,
@@ -92,7 +94,7 @@ impl TrivialMovePicker {
                     },
                 ],
                 target_level: self.target_level,
-                target_sub_level_id: 0,
+                ..Default::default()
             });
         }
 
diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
index 3d8b0f5718e47..9f84b99453f17 100644
--- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
@@ -199,6 +199,8 @@ impl TtlReclaimCompactionPicker {
         });
 
         Some(CompactionInput {
+            select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+            total_file_count: select_input_ssts.len() as _,
             input_levels: vec![
                 InputLevel {
                     level_idx: reclaimed_level.level_idx,
@@ -212,7 +214,7 @@ impl TtlReclaimCompactionPicker {
                 },
             ],
             target_level: reclaimed_level.level_idx as usize,
-            target_sub_level_id: 0,
+            ..Default::default()
         })
     }
 }
diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs
index 20b394d9b0480..c6dd5e2f82387 100644
--- a/src/meta/src/hummock/compactor_manager.rs
+++ b/src/meta/src/hummock/compactor_manager.rs
@@ -29,7 +29,6 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 use crate::manager::MetaSrvEnv;
 use crate::model::MetadataModel;
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
 pub type CompactorManagerRef = Arc<CompactorManager>;
@@ -125,7 +124,7 @@ pub struct CompactorManagerInner {
 }
 
 impl CompactorManagerInner {
-    pub async fn with_meta<S: MetaStore>(env: MetaSrvEnv<S>) -> MetaResult<Self> {
+    pub async fn with_meta(env: MetaSrvEnv) -> MetaResult<Self> {
         // Retrieve the existing task assignments from metastore.
         let task_assignment = CompactTaskAssignment::list(env.meta_store()).await?;
         let mut manager = Self {
@@ -370,7 +369,7 @@ pub struct CompactorManager {
 }
 
 impl CompactorManager {
-    pub async fn with_meta<S: MetaStore>(env: MetaSrvEnv<S>) -> MetaResult<Self> {
+    pub async fn with_meta(env: MetaSrvEnv) -> MetaResult<Self> {
         let inner = CompactorManagerInner::with_meta(env).await?;
 
         Ok(Self {
diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs
index 003e895ab8bea..d0c8428abedc5 100644
--- a/src/meta/src/hummock/manager/checkpoint.rs
+++ b/src/meta/src/hummock/manager/checkpoint.rs
@@ -33,10 +33,7 @@ const HUMMOCK_INIT_FLAG_KEY: &[u8] = b"hummock_init_flag";
 
 /// A hummock version checkpoint compacts previous hummock version delta logs, and stores stale
 /// objects from those delta logs.
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     /// # Panics
     /// if checkpoint is not found.
     pub(crate) async fn read_checkpoint(&self) -> Result<HummockVersionCheckpoint> {
diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs
index f46a3d47fa42a..db34da26e62fd 100644
--- a/src/meta/src/hummock/manager/compaction.rs
+++ b/src/meta/src/hummock/manager/compaction.rs
@@ -22,7 +22,6 @@ use risingwave_pb::hummock::{CompactStatus as PbCompactStatus, CompactTaskAssign
 use crate::hummock::compaction::CompactStatus;
 use crate::hummock::manager::read_lock;
 use crate::hummock::HummockManager;
-use crate::storage::MetaStore;
 
 #[derive(Default)]
 pub struct Compaction {
@@ -34,10 +33,7 @@ pub struct Compaction {
     pub deterministic_mode: bool,
 }
 
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     #[named]
     pub async fn get_assigned_compact_task_num(&self) -> u64 {
         read_lock!(self, compaction)
diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs
index b496a491b009d..8fa1aea32115f 100644
--- a/src/meta/src/hummock/manager/compaction_group_manager.rs
+++ b/src/meta/src/hummock/manager/compaction_group_manager.rs
@@ -50,9 +50,9 @@ use crate::model::{
 };
 use crate::storage::{MetaStore, Transaction};
 
-impl<S: MetaStore> HummockManager<S> {
+impl HummockManager {
     pub(super) async fn build_compaction_group_manager(
-        env: &MetaSrvEnv<S>,
+        env: &MetaSrvEnv,
     ) -> Result<RwLock<CompactionGroupManager>> {
         let default_config = match env.opts.compaction_config.as_ref() {
             None => CompactionConfigBuilder::new().build(),
@@ -62,7 +62,7 @@ impl<S: MetaStore> HummockManager<S> {
     }
 
     pub(super) async fn build_compaction_group_manager_with_config(
-        env: &MetaSrvEnv<S>,
+        env: &MetaSrvEnv,
         default_config: CompactionConfig,
     ) -> Result<RwLock<CompactionGroupManager>> {
         let compaction_group_manager = RwLock::new(CompactionGroupManager {
@@ -149,8 +149,8 @@ impl<S: MetaStore> HummockManager<S> {
     }
 
     /// Unregisters stale members and groups
-    /// The caller should ensure [`table_fragments_list`] remain unchanged during [`purge`].
-    /// Currently [`purge`] is only called during meta service start ups.
+    /// The caller should ensure `table_fragments_list` remain unchanged during `purge`.
+    /// Currently `purge` is only called during meta service start ups.
     #[named]
     pub async fn purge(&self, valid_ids: &[u32]) -> Result<()> {
         let registered_members =
@@ -165,8 +165,8 @@ impl<S: MetaStore> HummockManager<S> {
         Ok(())
     }
 
-    /// Prefer using [`register_table_fragments`].
-    /// Use [`register_table_ids`] only when [`TableFragments`] is unavailable.
+    /// Prefer using `register_table_fragments`.
+    /// Use `register_table_ids` only when [`TableFragments`] is unavailable.
     /// The implementation acquires `versioning` lock.
     #[named]
     pub async fn register_table_ids(
@@ -265,8 +265,8 @@ impl<S: MetaStore> HummockManager<S> {
         Ok(())
     }
 
-    /// Prefer using [`unregister_table_fragments_vec`].
-    /// Only Use [`unregister_table_ids`] only when [`TableFragments`] is unavailable.
+    /// Prefer using `unregister_table_fragments_vec`.
+    /// Only use `unregister_table_ids` when [`TableFragments`] is unavailable.
     /// The implementation acquires `versioning` lock and `compaction_group_manager` lock.
     #[named]
     pub async fn unregister_table_ids(&self, table_ids: &[StateTableId]) -> Result<()> {
diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs
index 3462058a93fda..21751bb968421 100644
--- a/src/meta/src/hummock/manager/context.rs
+++ b/src/meta/src/hummock/manager/context.rs
@@ -30,12 +30,9 @@ use crate::hummock::manager::{
 use crate::hummock::HummockManager;
 use crate::manager::META_NODE_ID;
 use crate::model::{BTreeMapTransaction, ValTransaction};
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::Transaction;
 
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     /// Release resources pinned by these contexts, including:
     /// - Version
     /// - Snapshot
diff --git a/src/meta/src/hummock/manager/gc.rs b/src/meta/src/hummock/manager/gc.rs
index e84bda80cacf9..5533d9be68e85 100644
--- a/src/meta/src/hummock/manager/gc.rs
+++ b/src/meta/src/hummock/manager/gc.rs
@@ -31,12 +31,9 @@ use crate::hummock::manager::{commit_multi_var, read_lock, write_lock, ResponseE
 use crate::hummock::HummockManager;
 use crate::manager::ClusterManagerRef;
 use crate::model::{BTreeMapTransaction, ValTransaction};
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::Transaction;
 
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     /// Gets SST objects that is safe to be deleted from object store.
     #[named]
     pub async fn get_objects_to_delete(&self) -> Vec<HummockSstableObjectId> {
@@ -204,13 +201,10 @@ where
 ///
 /// Returns a global GC watermark. The watermark only guards SSTs created before this
 /// invocation.
-pub async fn collect_global_gc_watermark<S>(
-    cluster_manager: ClusterManagerRef<S>,
+pub async fn collect_global_gc_watermark(
+    cluster_manager: ClusterManagerRef,
     spin_interval: Duration,
-) -> Result<HummockSstableObjectId>
-where
-    S: MetaStore,
-{
+) -> Result<HummockSstableObjectId> {
     let mut global_watermark = HummockSstableObjectId::MAX;
     let workers = [
         cluster_manager.list_active_streaming_compute_nodes().await,
diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs
index 134b410fb8a59..03d75112d01fe 100644
--- a/src/meta/src/hummock/manager/mod.rs
+++ b/src/meta/src/hummock/manager/mod.rs
@@ -83,7 +83,7 @@ use crate::model::{
     VarTransaction,
 };
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::{MetaStore, MetaStoreRef, Transaction};
 
 mod compaction_group_manager;
 mod context;
@@ -107,12 +107,12 @@ const HISTORY_TABLE_INFO_STATISTIC_TIME: usize = 240;
 // - Make changes on the ValTransaction.
 // - Call `commit_multi_var` to commit the changes via meta store transaction. If transaction
 //   succeeds, the in-mem state will be updated by the way.
-pub struct HummockManager<S: MetaStore> {
-    pub env: MetaSrvEnv<S>,
-    pub cluster_manager: ClusterManagerRef<S>,
-    catalog_manager: CatalogManagerRef<S>,
+pub struct HummockManager {
+    pub env: MetaSrvEnv,
+    pub cluster_manager: ClusterManagerRef,
+    catalog_manager: CatalogManagerRef,
 
-    fragment_manager: FragmentManagerRef<S>,
+    fragment_manager: FragmentManagerRef,
     // `CompactionGroupManager` manages `CompactionGroup`'s members.
     // Note that all hummock state store user should register to `CompactionGroupManager`. It
     // includes all state tables of streaming jobs except sink.
@@ -143,7 +143,7 @@ pub struct HummockManager<S: MetaStore> {
     pub compaction_state: CompactionState,
 }
 
-pub type HummockManagerRef<S> = Arc<HummockManager<S>>;
+pub type HummockManagerRef = Arc<HummockManager>;
 
 /// Commit multiple `ValTransaction`s to state store and upon success update the local in-mem state
 /// by the way
@@ -252,22 +252,19 @@ pub enum CompactionResumeTrigger {
     TaskReport { original_task_num: usize },
 }
 
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     pub(crate) async fn new(
-        env: MetaSrvEnv<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
+        env: MetaSrvEnv,
+        cluster_manager: ClusterManagerRef,
+        fragment_manager: FragmentManagerRef,
         metrics: Arc<MetaMetrics>,
         compactor_manager: CompactorManagerRef,
-        catalog_manager: CatalogManagerRef<S>,
+        catalog_manager: CatalogManagerRef,
         compactor_streams_change_tx: UnboundedSender<(
             u32,
             Streaming<SubscribeCompactionEventRequest>,
         )>,
-    ) -> Result<HummockManagerRef<S>> {
+    ) -> Result<HummockManagerRef> {
         let compaction_group_manager = Self::build_compaction_group_manager(&env).await?;
         Self::new_impl(
             env,
@@ -284,9 +281,9 @@ where
 
     #[cfg(any(test, feature = "test"))]
     pub(super) async fn with_config(
-        env: MetaSrvEnv<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
+        env: MetaSrvEnv,
+        cluster_manager: ClusterManagerRef,
+        fragment_manager: FragmentManagerRef,
         metrics: Arc<MetaMetrics>,
         compactor_manager: CompactorManagerRef,
         config: CompactionConfig,
@@ -294,7 +291,7 @@ where
             u32,
             Streaming<SubscribeCompactionEventRequest>,
         )>,
-    ) -> HummockManagerRef<S> {
+    ) -> HummockManagerRef {
         use crate::manager::CatalogManager;
         let compaction_group_manager =
             Self::build_compaction_group_manager_with_config(&env, config)
@@ -316,18 +313,18 @@ where
     }
 
     async fn new_impl(
-        env: MetaSrvEnv<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
+        env: MetaSrvEnv,
+        cluster_manager: ClusterManagerRef,
+        fragment_manager: FragmentManagerRef,
         metrics: Arc<MetaMetrics>,
         compactor_manager: CompactorManagerRef,
         compaction_group_manager: tokio::sync::RwLock<CompactionGroupManager>,
-        catalog_manager: CatalogManagerRef<S>,
+        catalog_manager: CatalogManagerRef,
         compactor_streams_change_tx: UnboundedSender<(
             u32,
             Streaming<SubscribeCompactionEventRequest>,
         )>,
-    ) -> Result<HummockManagerRef<S>> {
+    ) -> Result<HummockManagerRef> {
         let sys_params_manager = env.system_params_manager();
         let sys_params = sys_params_manager.get_params().await;
         let state_store_url = sys_params.state_store();
@@ -540,7 +537,7 @@ where
     /// call `release_contexts` even if it has removed `context_id` from cluster manager.
     async fn commit_trx(
         &self,
-        meta_store: &S,
+        meta_store: &MetaStoreRef,
         trx: Transaction,
         context_id: Option<HummockContextId>,
     ) -> Result<()> {
@@ -1926,7 +1923,7 @@ where
         assignment_ref.get(&task_id).cloned()
     }
 
-    pub fn cluster_manager(&self) -> &ClusterManagerRef<S> {
+    pub fn cluster_manager(&self) -> &ClusterManagerRef {
         &self.cluster_manager
     }
 
@@ -2775,34 +2772,47 @@ fn gen_version_delta<'a>(
         .or_default()
         .group_deltas;
     let mut gc_object_ids = vec![];
+    let mut removed_table_ids_map: BTreeMap<u32, Vec<u64>> = BTreeMap::default();
+
     for level in &compact_task.input_ssts {
+        let level_idx = level.level_idx;
+        let mut removed_table_ids = level
+            .table_infos
+            .iter()
+            .map(|sst| {
+                let object_id = sst.get_object_id();
+                let sst_id = sst.get_sst_id();
+                if !trivial_move
+                    && drop_sst(
+                        branched_ssts,
+                        compact_task.compaction_group_id,
+                        object_id,
+                        sst_id,
+                    )
+                {
+                    gc_object_ids.push(object_id);
+                }
+                sst_id
+            })
+            .collect_vec();
+
+        removed_table_ids_map
+            .entry(level_idx)
+            .or_default()
+            .append(&mut removed_table_ids);
+    }
+
+    for (level_idx, removed_table_ids) in removed_table_ids_map {
         let group_delta = GroupDelta {
             delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                level_idx: level.level_idx,
-                removed_table_ids: level
-                    .table_infos
-                    .iter()
-                    .map(|sst| {
-                        let object_id = sst.get_object_id();
-                        let sst_id = sst.get_sst_id();
-                        if !trivial_move
-                            && drop_sst(
-                                branched_ssts,
-                                compact_task.compaction_group_id,
-                                object_id,
-                                sst_id,
-                            )
-                        {
-                            gc_object_ids.push(object_id);
-                        }
-                        sst_id
-                    })
-                    .collect_vec(),
+                level_idx,
+                removed_table_ids,
                 ..Default::default()
             })),
         };
         group_deltas.push(group_delta);
     }
+
     let group_delta = GroupDelta {
         delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
             level_idx: compact_task.target_level,
diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs
index 9604a060566b0..596149df3b8aa 100644
--- a/src/meta/src/hummock/manager/tests.rs
+++ b/src/meta/src/hummock/manager/tests.rs
@@ -17,6 +17,7 @@ use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashMap};
 
 use itertools::Itertools;
+use prometheus::Registry;
 use risingwave_common::util::epoch::INVALID_EPOCH;
 use risingwave_hummock_sdk::compact::compact_task_to_string;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
@@ -37,6 +38,7 @@ use risingwave_pb::hummock::{
 };
 use risingwave_pb::meta::add_worker_node_request::Property;
 
+use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
 use crate::hummock::compaction::{
     default_level_selector, CompactStatus, LevelSelector, ManualCompactionOption,
     SpaceReclaimCompactionSelector,
@@ -46,7 +48,7 @@ use crate::hummock::test_utils::*;
 use crate::hummock::{HummockManager, HummockManagerRef};
 use crate::manager::WorkerId;
 use crate::model::MetadataModel;
-use crate::storage::{MemStore, MetaStore};
+use crate::rpc::metrics::MetaMetrics;
 
 fn pin_versions_sum(pin_versions: &[HummockPinnedVersion]) -> usize {
     pin_versions.iter().len()
@@ -481,7 +483,7 @@ async fn test_hummock_manager_basic() {
     let mut epoch = 1;
     let mut register_log_count = 0;
     let mut commit_log_count = 0;
-    let commit_one = |epoch: HummockEpoch, hummock_manager: HummockManagerRef<MemStore>| async move {
+    let commit_one = |epoch: HummockEpoch, hummock_manager: HummockManagerRef| async move {
         let original_tables = generate_test_tables(epoch, get_sst_ids(&hummock_manager, 2).await);
         register_sstable_infos_to_compaction_group(
             &hummock_manager,
@@ -1315,8 +1317,8 @@ async fn test_split_compaction_group_on_commit() {
     );
 }
 
-async fn get_branched_ssts<S: MetaStore>(
-    hummock_manager: &HummockManager<S>,
+async fn get_branched_ssts(
+    hummock_manager: &HummockManager,
 ) -> BTreeMap<HummockSstableObjectId, BranchedSstInfo> {
     hummock_manager
         .versioning
@@ -1694,8 +1696,8 @@ async fn test_split_compaction_group_trivial_expired() {
     assert!(!ret);
 }
 
-async fn get_manual_compact_task<S: MetaStore>(
-    hummock_manager: &HummockManager<S>,
+async fn get_manual_compact_task(
+    hummock_manager: &HummockManager,
     context_id: HummockContextId,
 ) -> CompactTask {
     hummock_manager.compactor_manager.add_compactor(context_id);
@@ -2044,14 +2046,16 @@ async fn test_move_tables_between_compaction_group() {
 
 #[tokio::test]
 async fn test_gc_stats() {
-    let (_env, hummock_manager, _, worker_node) = setup_compute_env(80).await;
+    let config = CompactionConfigBuilder::new()
+        .level0_tier_compact_file_number(1)
+        .level0_max_compact_file_number(130)
+        .level0_sub_level_compact_level_count(1)
+        .level0_overlapping_sub_level_compact_level_count(1)
+        .build();
+    let registry = Registry::new();
+    let (_env, hummock_manager, _, worker_node) =
+        setup_compute_env_with_metric(80, config, Some(MetaMetrics::for_test(&registry))).await;
     let context_id = worker_node.id;
-    hummock_manager.metrics.stale_object_size.set(0);
-    hummock_manager.metrics.stale_object_count.set(0);
-    hummock_manager.metrics.old_version_object_size.set(0);
-    hummock_manager.metrics.old_version_object_count.set(0);
-    hummock_manager.metrics.current_version_object_count.set(0);
-    hummock_manager.metrics.current_version_object_size.set(0);
     let assert_eq_gc_stats = |stale_object_size,
                               stale_object_count,
                               old_version_object_size,
diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs
index 42cef3a7d1b12..09a331fa3d9f5 100644
--- a/src/meta/src/hummock/manager/versioning.rs
+++ b/src/meta/src/hummock/manager/versioning.rs
@@ -39,7 +39,6 @@ use crate::hummock::manager::{read_lock, write_lock};
 use crate::hummock::metrics_utils::{trigger_safepoint_stat, trigger_write_stop_stats};
 use crate::hummock::model::CompactionGroup;
 use crate::hummock::HummockManager;
-use crate::storage::MetaStore;
 
 /// `HummockVersionSafePoint` prevents hummock versions GE than it from being GC.
 /// It's used by meta node itself to temporarily pin versions.
@@ -161,10 +160,7 @@ impl Versioning {
     }
 }
 
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     #[named]
     pub async fn list_pinned_version(&self) -> Vec<HummockPinnedVersion> {
         read_lock!(self, versioning)
diff --git a/src/meta/src/hummock/manager/worker.rs b/src/meta/src/hummock/manager/worker.rs
index 5db54177b000b..8a43ddc87247b 100644
--- a/src/meta/src/hummock/manager/worker.rs
+++ b/src/meta/src/hummock/manager/worker.rs
@@ -23,7 +23,6 @@ use tokio_retry::strategy::{jitter, ExponentialBackoff};
 use crate::hummock::utils::RetryableError;
 use crate::hummock::{HummockManager, HummockManagerRef};
 use crate::manager::LocalNotification;
-use crate::storage::MetaStore;
 
 pub type HummockManagerEventSender = tokio::sync::mpsc::UnboundedSender<HummockManagerEvent>;
 pub type HummockManagerEventReceiver = tokio::sync::mpsc::UnboundedReceiver<HummockManagerEvent>;
@@ -34,12 +33,9 @@ pub enum HummockManagerEvent {
     Shutdown,
 }
 
-impl<S> HummockManager<S>
-where
-    S: MetaStore,
-{
+impl HummockManager {
     pub(crate) async fn start_worker(
-        self: &HummockManagerRef<S>,
+        self: &HummockManagerRef,
         mut receiver: HummockManagerEventReceiver,
     ) -> JoinHandle<()> {
         let (local_notification_tx, mut local_notification_rx) =
diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs
index 9ca7f24686916..915beee8e3a3f 100644
--- a/src/meta/src/hummock/mock_hummock_meta_client.rs
+++ b/src/meta/src/hummock/mock_hummock_meta_client.rs
@@ -42,10 +42,9 @@ use crate::hummock::compaction::{
     default_level_selector, LevelSelector, SpaceReclaimCompactionSelector,
 };
 use crate::hummock::HummockManager;
-use crate::storage::MemStore;
 
 pub struct MockHummockMetaClient {
-    hummock_manager: Arc<HummockManager<MemStore>>,
+    hummock_manager: Arc<HummockManager>,
     context_id: HummockContextId,
     compact_context_id: AtomicU32,
     // used for hummock replay to avoid collision with existing sst files
@@ -54,7 +53,7 @@ pub struct MockHummockMetaClient {
 
 impl MockHummockMetaClient {
     pub fn new(
-        hummock_manager: Arc<HummockManager<MemStore>>,
+        hummock_manager: Arc<HummockManager>,
         context_id: HummockContextId,
     ) -> MockHummockMetaClient {
         MockHummockMetaClient {
@@ -66,7 +65,7 @@ impl MockHummockMetaClient {
     }
 
     pub fn with_sst_offset(
-        hummock_manager: Arc<HummockManager<MemStore>>,
+        hummock_manager: Arc<HummockManager>,
         context_id: HummockContextId,
         sst_offset: u64,
     ) -> Self {
@@ -282,7 +281,7 @@ impl HummockMetaClient for MockHummockMetaClient {
 }
 
 impl MockHummockMetaClient {
-    pub fn hummock_manager_ref(&self) -> Arc<HummockManager<MemStore>> {
+    pub fn hummock_manager_ref(&self) -> Arc<HummockManager> {
         self.hummock_manager.clone()
     }
 }
diff --git a/src/meta/src/hummock/mod.rs b/src/meta/src/hummock/mod.rs
index 5bc5a5cff45c3..65daf589fb986 100644
--- a/src/meta/src/hummock/mod.rs
+++ b/src/meta/src/hummock/mod.rs
@@ -36,18 +36,14 @@ use tokio::sync::oneshot::Sender;
 use tokio::task::JoinHandle;
 pub use vacuum::*;
 
-use crate::storage::MetaStore;
 use crate::MetaOpts;
 
 /// Start hummock's asynchronous tasks.
-pub fn start_hummock_workers<S>(
-    hummock_manager: HummockManagerRef<S>,
-    vacuum_manager: VacuumManagerRef<S>,
+pub fn start_hummock_workers(
+    hummock_manager: HummockManagerRef,
+    vacuum_manager: VacuumManagerRef,
     meta_opts: &MetaOpts,
-) -> Vec<(JoinHandle<()>, Sender<()>)>
-where
-    S: MetaStore,
-{
+) -> Vec<(JoinHandle<()>, Sender<()>)> {
     // These critical tasks are put in their own timer loop deliberately, to avoid long-running ones
     // from blocking others.
     let workers = vec![
@@ -69,13 +65,10 @@ where
 }
 
 /// Starts a task to periodically vacuum stale metadata.
-pub fn start_vacuum_metadata_loop<S>(
-    vacuum: VacuumManagerRef<S>,
+pub fn start_vacuum_metadata_loop(
+    vacuum: VacuumManagerRef,
     interval: Duration,
-) -> (JoinHandle<()>, Sender<()>)
-where
-    S: MetaStore,
-{
+) -> (JoinHandle<()>, Sender<()>) {
     let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
     let join_handle = tokio::spawn(async move {
         let mut min_trigger_interval = tokio::time::interval(interval);
@@ -99,13 +92,10 @@ where
 }
 
 /// Starts a task to periodically vacuum stale objects.
-pub fn start_vacuum_object_loop<S>(
-    vacuum: VacuumManagerRef<S>,
+pub fn start_vacuum_object_loop(
+    vacuum: VacuumManagerRef,
     interval: Duration,
-) -> (JoinHandle<()>, Sender<()>)
-where
-    S: MetaStore,
-{
+) -> (JoinHandle<()>, Sender<()>) {
     let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
     let join_handle = tokio::spawn(async move {
         let mut min_trigger_interval = tokio::time::interval(interval);
@@ -128,8 +118,8 @@ where
     (join_handle, shutdown_tx)
 }
 
-pub fn start_checkpoint_loop<S: MetaStore>(
-    hummock_manager: HummockManagerRef<S>,
+pub fn start_checkpoint_loop(
+    hummock_manager: HummockManagerRef,
     interval: Duration,
     min_delta_log_num: u64,
 ) -> (JoinHandle<()>, Sender<()>) {
diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs
index 749096c64151b..632d56ca2c400 100644
--- a/src/meta/src/hummock/test_utils.rs
+++ b/src/meta/src/hummock/test_utils.rs
@@ -37,7 +37,6 @@ use crate::manager::{
     ClusterManager, ClusterManagerRef, FragmentManager, MetaSrvEnv, META_NODE_ID,
 };
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::{MemStore, MetaStore};
 
 pub fn to_local_sstable_info(ssts: &[SstableInfo]) -> Vec<LocalSstableInfo> {
     ssts.iter()
@@ -51,13 +50,10 @@ pub fn to_local_sstable_info(ssts: &[SstableInfo]) -> Vec<LocalSstableInfo> {
 }
 
 #[cfg(test)]
-pub async fn add_test_tables<S>(
-    hummock_manager: &HummockManager<S>,
+pub async fn add_test_tables(
+    hummock_manager: &HummockManager,
     context_id: HummockContextId,
-) -> Vec<Vec<SstableInfo>>
-where
-    S: MetaStore,
-{
+) -> Vec<Vec<SstableInfo>> {
     // Increase version by 2.
     let mut epoch: u64 = 1;
     let sstable_ids = get_sst_ids(hummock_manager, 3).await;
@@ -207,13 +203,11 @@ pub fn generate_test_tables(epoch: u64, sst_ids: Vec<HummockSstableObjectId>) ->
     sst_info
 }
 
-pub async fn register_sstable_infos_to_compaction_group<S>(
-    compaction_group_manager_ref: &HummockManager<S>,
+pub async fn register_sstable_infos_to_compaction_group(
+    compaction_group_manager_ref: &HummockManager,
     sstable_infos: &[SstableInfo],
     compaction_group_id: CompactionGroupId,
-) where
-    S: MetaStore,
-{
+) {
     let table_ids = sstable_infos
         .iter()
         .flat_map(|sstable_info| &sstable_info.table_ids)
@@ -229,13 +223,11 @@ pub async fn register_sstable_infos_to_compaction_group<S>(
     .await;
 }
 
-pub async fn register_table_ids_to_compaction_group<S>(
-    hummock_manager_ref: &HummockManager<S>,
+pub async fn register_table_ids_to_compaction_group(
+    hummock_manager_ref: &HummockManager,
     table_ids: &[u32],
     compaction_group_id: CompactionGroupId,
-) where
-    S: MetaStore,
-{
+) {
     hummock_manager_ref
         .register_table_ids(
             &table_ids
@@ -247,12 +239,10 @@ pub async fn register_table_ids_to_compaction_group<S>(
         .unwrap();
 }
 
-pub async fn unregister_table_ids_from_compaction_group<S>(
-    hummock_manager_ref: &HummockManager<S>,
+pub async fn unregister_table_ids_from_compaction_group(
+    hummock_manager_ref: &HummockManager,
     table_ids: &[u32],
-) where
-    S: MetaStore,
-{
+) {
     hummock_manager_ref
         .unregister_table_ids(table_ids)
         .await
@@ -300,15 +290,19 @@ pub fn get_sorted_committed_object_ids(
         .sorted()
         .collect_vec()
 }
+
 pub async fn setup_compute_env_with_config(
     port: i32,
     config: CompactionConfig,
-) -> (
-    MetaSrvEnv<MemStore>,
-    HummockManagerRef<MemStore>,
-    ClusterManagerRef<MemStore>,
-    WorkerNode,
-) {
+) -> (MetaSrvEnv, HummockManagerRef, ClusterManagerRef, WorkerNode) {
+    setup_compute_env_with_metric(port, config, None).await
+}
+
+pub async fn setup_compute_env_with_metric(
+    port: i32,
+    config: CompactionConfig,
+    meta_metric: Option<MetaMetrics>,
+) -> (MetaSrvEnv, HummockManagerRef, ClusterManagerRef, WorkerNode) {
     let env = MetaSrvEnv::for_test().await;
     let cluster_manager = Arc::new(
         ClusterManager::new(env.clone(), Duration::from_secs(1))
@@ -326,7 +320,7 @@ pub async fn setup_compute_env_with_config(
         env.clone(),
         cluster_manager.clone(),
         fragment_manager,
-        Arc::new(MetaMetrics::default()),
+        Arc::new(meta_metric.unwrap_or_default()),
         compactor_manager,
         config,
         compactor_streams_change_tx,
@@ -355,12 +349,7 @@ pub async fn setup_compute_env_with_config(
 
 pub async fn setup_compute_env(
     port: i32,
-) -> (
-    MetaSrvEnv<MemStore>,
-    HummockManagerRef<MemStore>,
-    ClusterManagerRef<MemStore>,
-    WorkerNode,
-) {
+) -> (MetaSrvEnv, HummockManagerRef, ClusterManagerRef, WorkerNode) {
     let config = CompactionConfigBuilder::new()
         .level0_tier_compact_file_number(1)
         .level0_max_compact_file_number(130)
@@ -370,25 +359,19 @@ pub async fn setup_compute_env(
     setup_compute_env_with_config(port, config).await
 }
 
-pub async fn get_sst_ids<S>(
-    hummock_manager: &HummockManager<S>,
+pub async fn get_sst_ids(
+    hummock_manager: &HummockManager,
     number: u32,
-) -> Vec<HummockSstableObjectId>
-where
-    S: MetaStore,
-{
+) -> Vec<HummockSstableObjectId> {
     let range = hummock_manager.get_new_sst_ids(number).await.unwrap();
     (range.start_id..range.end_id).collect_vec()
 }
 
-pub async fn commit_from_meta_node<S>(
-    hummock_manager_ref: &HummockManager<S>,
+pub async fn commit_from_meta_node(
+    hummock_manager_ref: &HummockManager,
     epoch: HummockEpoch,
     ssts: Vec<LocalSstableInfo>,
-) -> crate::hummock::error::Result<Option<HummockSnapshot>>
-where
-    S: MetaStore,
-{
+) -> crate::hummock::error::Result<Option<HummockSnapshot>> {
     let sst_to_worker = ssts
         .iter()
         .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), META_NODE_ID))
@@ -398,14 +381,11 @@ where
         .await
 }
 
-pub async fn add_ssts<S>(
+pub async fn add_ssts(
     epoch: HummockEpoch,
-    hummock_manager: &HummockManager<S>,
+    hummock_manager: &HummockManager,
     context_id: HummockContextId,
-) -> Vec<SstableInfo>
-where
-    S: MetaStore,
-{
+) -> Vec<SstableInfo> {
     let table_ids = get_sst_ids(hummock_manager, 3).await;
     let test_tables = generate_test_sstables_with_table_id(epoch, 1, table_ids);
     let ssts = to_local_sstable_info(&test_tables);
diff --git a/src/meta/src/hummock/vacuum.rs b/src/meta/src/hummock/vacuum.rs
index 490111b652506..31f4651d6fdfd 100644
--- a/src/meta/src/hummock/vacuum.rs
+++ b/src/meta/src/hummock/vacuum.rs
@@ -25,29 +25,25 @@ use super::CompactorManagerRef;
 use crate::backup_restore::BackupManagerRef;
 use crate::hummock::HummockManagerRef;
 use crate::manager::MetaSrvEnv;
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
-pub type VacuumManagerRef<S> = Arc<VacuumManager<S>>;
+pub type VacuumManagerRef = Arc<VacuumManager>;
 
-pub struct VacuumManager<S: MetaStore> {
-    env: MetaSrvEnv<S>,
-    hummock_manager: HummockManagerRef<S>,
-    backup_manager: BackupManagerRef<S>,
+pub struct VacuumManager {
+    env: MetaSrvEnv,
+    hummock_manager: HummockManagerRef,
+    backup_manager: BackupManagerRef,
     /// Use the CompactorManager to dispatch VacuumTask.
     compactor_manager: CompactorManagerRef,
     /// SST object ids which have been dispatched to vacuum nodes but are not replied yet.
     pending_object_ids: parking_lot::RwLock<HashSet<HummockSstableObjectId>>,
 }
 
-impl<S> VacuumManager<S>
-where
-    S: MetaStore,
-{
+impl VacuumManager {
     pub fn new(
-        env: MetaSrvEnv<S>,
-        hummock_manager: HummockManagerRef<S>,
-        backup_manager: BackupManagerRef<S>,
+        env: MetaSrvEnv,
+        hummock_manager: HummockManagerRef,
+        backup_manager: BackupManagerRef,
         compactor_manager: CompactorManagerRef,
     ) -> Self {
         Self {
diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs
index b90f2715ccc00..92d3c571f57c5 100644
--- a/src/meta/src/lib.rs
+++ b/src/meta/src/lib.rs
@@ -244,6 +244,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
             dashboard_addr,
             ui_path: opts.dashboard_ui_path,
         };
+
         let (mut join_handle, leader_lost_handle, shutdown_send) = rpc_serve(
             add_info,
             backend,
diff --git a/src/meta/src/manager/catalog/database.rs b/src/meta/src/manager/catalog/database.rs
index 86b9b0a2d947c..705474dd27a6d 100644
--- a/src/meta/src/manager/catalog/database.rs
+++ b/src/meta/src/manager/catalog/database.rs
@@ -25,7 +25,6 @@ use risingwave_pb::catalog::{
 use super::{ConnectionId, DatabaseId, FunctionId, RelationId, SchemaId, SinkId, SourceId, ViewId};
 use crate::manager::{IndexId, MetaSrvEnv, TableId};
 use crate::model::MetadataModel;
-use crate::storage::MetaStore;
 use crate::{MetaError, MetaResult};
 
 pub type Catalog = (
@@ -79,7 +78,7 @@ pub struct DatabaseManager {
 }
 
 impl DatabaseManager {
-    pub async fn new<S: MetaStore>(env: MetaSrvEnv<S>) -> MetaResult<Self> {
+    pub async fn new(env: MetaSrvEnv) -> MetaResult<Self> {
         let databases = Database::list(env.meta_store()).await?;
         let schemas = Schema::list(env.meta_store()).await?;
         let sources = Source::list(env.meta_store()).await?;
diff --git a/src/meta/src/manager/catalog/fragment.rs b/src/meta/src/manager/catalog/fragment.rs
index 5a8fd330578b3..1a74608c848a1 100644
--- a/src/meta/src/manager/catalog/fragment.rs
+++ b/src/meta/src/manager/catalog/fragment.rs
@@ -41,7 +41,7 @@ use crate::model::{
     ActorId, BTreeMapTransaction, FragmentId, MetadataModel, MigrationPlan, TableFragments,
     ValTransaction,
 };
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::Transaction;
 use crate::stream::{SplitAssignment, TableRevision};
 use crate::MetaResult;
 
@@ -103,8 +103,8 @@ impl FragmentManagerCore {
 }
 
 /// `FragmentManager` stores definition and status of fragment as well as the actors inside.
-pub struct FragmentManager<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct FragmentManager {
+    env: MetaSrvEnv,
 
     core: RwLock<FragmentManagerCore>,
 }
@@ -117,13 +117,10 @@ pub struct ActorInfos {
     pub barrier_inject_actor_maps: HashMap<WorkerId, Vec<ActorId>>,
 }
 
-pub type FragmentManagerRef<S> = Arc<FragmentManager<S>>;
+pub type FragmentManagerRef = Arc<FragmentManager>;
 
-impl<S: MetaStore> FragmentManager<S>
-where
-    S: MetaStore,
-{
-    pub async fn new(env: MetaSrvEnv<S>) -> MetaResult<Self> {
+impl FragmentManager {
+    pub async fn new(env: MetaSrvEnv) -> MetaResult<Self> {
         let table_fragments = TableFragments::list(env.meta_store()).await?;
 
         let table_fragments = table_fragments
diff --git a/src/meta/src/manager/catalog/mod.rs b/src/meta/src/manager/catalog/mod.rs
index a6552f2126a11..0e3fa33d0eec1 100644
--- a/src/meta/src/manager/catalog/mod.rs
+++ b/src/meta/src/manager/catalog/mod.rs
@@ -45,7 +45,7 @@ use user::*;
 
 use crate::manager::{IdCategory, MetaSrvEnv, NotificationVersion, StreamingJob};
 use crate::model::{BTreeMapTransaction, MetadataModel, ValTransaction};
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::Transaction;
 use crate::{MetaError, MetaResult};
 
 pub type DatabaseId = u32;
@@ -75,6 +75,7 @@ macro_rules! commit_meta_with_trx {
     ($manager:expr, $trx:ident, $($val_txn:expr),*) => {
         {
             use tracing::Instrument;
+            use $crate::storage::meta_store::MetaStore;
             async {
                 // Apply the change in `ValTransaction` to trx
                 $(
@@ -123,7 +124,7 @@ use crate::manager::catalog::utils::{
 };
 use crate::rpc::ddl_controller::DropMode;
 
-pub type CatalogManagerRef<S> = Arc<CatalogManager<S>>;
+pub type CatalogManagerRef = Arc<CatalogManager>;
 
 /// `CatalogManager` manages database catalog information and user information, including
 /// authentication and privileges.
@@ -131,8 +132,8 @@ pub type CatalogManagerRef<S> = Arc<CatalogManager<S>>;
 /// It only has some basic validation for the user information.
 /// Other authorization relate to the current session user should be done in Frontend before passing
 /// to Meta.
-pub struct CatalogManager<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct CatalogManager {
+    env: MetaSrvEnv,
     core: Mutex<CatalogManagerCore>,
 }
 
@@ -142,18 +143,15 @@ pub struct CatalogManagerCore {
 }
 
 impl CatalogManagerCore {
-    async fn new<S: MetaStore>(env: MetaSrvEnv<S>) -> MetaResult<Self> {
+    async fn new(env: MetaSrvEnv) -> MetaResult<Self> {
         let database = DatabaseManager::new(env.clone()).await?;
         let user = UserManager::new(env.clone(), &database).await?;
         Ok(Self { database, user })
     }
 }
 
-impl<S> CatalogManager<S>
-where
-    S: MetaStore,
-{
-    pub async fn new(env: MetaSrvEnv<S>) -> MetaResult<Self> {
+impl CatalogManager {
+    pub async fn new(env: MetaSrvEnv) -> MetaResult<Self> {
         let core = Mutex::new(CatalogManagerCore::new(env.clone()).await?);
         let catalog_manager = Self { env, core };
         catalog_manager.init().await?;
@@ -172,10 +170,7 @@ where
 }
 
 // Database catalog related methods
-impl<S> CatalogManager<S>
-where
-    S: MetaStore,
-{
+impl CatalogManager {
     async fn init_database(&self) -> MetaResult<()> {
         let mut database = Database {
             name: DEFAULT_DATABASE_NAME.to_string(),
@@ -795,7 +790,7 @@ where
     pub async fn drop_relation(
         &self,
         relation: RelationIdEnum,
-        fragment_manager: FragmentManagerRef<S>,
+        fragment_manager: FragmentManagerRef,
         drop_mode: DropMode,
     ) -> MetaResult<(NotificationVersion, Vec<StreamingJobId>)> {
         let core = &mut *self.core.lock().await;
@@ -2257,10 +2252,7 @@ where
 }
 
 // User related methods
-impl<S> CatalogManager<S>
-where
-    S: MetaStore,
-{
+impl CatalogManager {
     async fn init_user(&self) -> MetaResult<()> {
         let core = &mut self.core.lock().await.user;
         for (user, id) in [
diff --git a/src/meta/src/manager/catalog/user.rs b/src/meta/src/manager/catalog/user.rs
index 88a52a11feabe..8037938937015 100644
--- a/src/meta/src/manager/catalog/user.rs
+++ b/src/meta/src/manager/catalog/user.rs
@@ -22,7 +22,6 @@ use super::database::DatabaseManager;
 use super::UserId;
 use crate::manager::MetaSrvEnv;
 use crate::model::MetadataModel;
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
 pub struct UserManager {
@@ -34,10 +33,7 @@ pub struct UserManager {
 }
 
 impl UserManager {
-    pub async fn new<S: MetaStore>(
-        env: MetaSrvEnv<S>,
-        database: &DatabaseManager,
-    ) -> MetaResult<Self> {
+    pub async fn new(env: MetaSrvEnv, database: &DatabaseManager) -> MetaResult<Self> {
         let users = UserInfo::list(env.meta_store()).await?;
         let user_info = BTreeMap::from_iter(users.into_iter().map(|user| (user.id, user)));
 
@@ -135,7 +131,7 @@ mod tests {
     use super::*;
     use crate::manager::{commit_meta, CatalogManager};
     use crate::model::{BTreeMapTransaction, ValTransaction};
-    use crate::storage::{MemStore, Transaction};
+    use crate::storage::Transaction;
 
     fn make_test_user(id: u32, name: &str) -> UserInfo {
         UserInfo {
@@ -402,7 +398,7 @@ mod tests {
         // Release all privileges with object.
         let user_core = &mut catalog_manager.core.lock().await.user;
         let mut users = BTreeMapTransaction::new(&mut user_core.user_info);
-        CatalogManager::<MemStore>::update_user_privileges(&mut users, &[object]);
+        CatalogManager::update_user_privileges(&mut users, &[object]);
         commit_meta!(&catalog_manager, users)?;
         let user = user_core.user_info.get(&test_user_id).unwrap();
         assert!(user.grant_privileges.is_empty());
diff --git a/src/meta/src/manager/cluster.rs b/src/meta/src/manager/cluster.rs
index 58a67a2da1f58..da5b4fce20711 100644
--- a/src/meta/src/manager/cluster.rs
+++ b/src/meta/src/manager/cluster.rs
@@ -32,12 +32,12 @@ use tokio::task::JoinHandle;
 
 use crate::manager::{IdCategory, LocalNotification, MetaSrvEnv};
 use crate::model::{MetadataModel, ValTransaction, VarTransaction, Worker, INVALID_EXPIRE_AT};
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::{MetaStore, MetaStoreRef, Transaction};
 use crate::{MetaError, MetaResult};
 
 pub type WorkerId = u32;
 pub type WorkerLocations = HashMap<WorkerId, WorkerNode>;
-pub type ClusterManagerRef<S> = Arc<ClusterManager<S>>;
+pub type ClusterManagerRef = Arc<ClusterManager>;
 
 #[derive(Clone, Debug)]
 pub struct WorkerKey(pub HostAddress);
@@ -61,19 +61,16 @@ impl Hash for WorkerKey {
 pub const META_NODE_ID: u32 = 0;
 
 /// [`ClusterManager`] manager cluster/worker meta data in [`MetaStore`].
-pub struct ClusterManager<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct ClusterManager {
+    env: MetaSrvEnv,
 
     max_heartbeat_interval: Duration,
 
     core: RwLock<ClusterManagerCore>,
 }
 
-impl<S> ClusterManager<S>
-where
-    S: MetaStore,
-{
-    pub async fn new(env: MetaSrvEnv<S>, max_heartbeat_interval: Duration) -> MetaResult<Self> {
+impl ClusterManager {
+    pub async fn new(env: MetaSrvEnv, max_heartbeat_interval: Duration) -> MetaResult<Self> {
         let core = ClusterManagerCore::new(env.meta_store_ref()).await?;
 
         Ok(Self {
@@ -333,7 +330,7 @@ where
     }
 
     pub fn start_heartbeat_checker(
-        cluster_manager: ClusterManagerRef<S>,
+        cluster_manager: ClusterManagerRef,
         check_interval: Duration,
     ) -> (JoinHandle<()>, Sender<()>) {
         let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
@@ -518,11 +515,8 @@ impl ClusterManagerCore {
     pub const MAX_WORKER_REUSABLE_ID_BITS: usize = 10;
     pub const MAX_WORKER_REUSABLE_ID_COUNT: usize = 1 << Self::MAX_WORKER_REUSABLE_ID_BITS;
 
-    async fn new<S>(meta_store: Arc<S>) -> MetaResult<Self>
-    where
-        S: MetaStore,
-    {
-        let mut workers = Worker::list(&*meta_store).await?;
+    async fn new(meta_store: MetaStoreRef) -> MetaResult<Self> {
+        let mut workers = Worker::list(&meta_store).await?;
 
         let used_transactional_ids: HashSet<_> = workers
             .iter()
@@ -729,7 +723,6 @@ impl ClusterManagerCore {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::storage::MemStore;
 
     #[tokio::test]
     async fn test_cluster_manager() -> MetaResult<()> {
@@ -889,10 +882,7 @@ mod tests {
         Ok(())
     }
 
-    async fn assert_cluster_manager(
-        cluster_manager: &ClusterManager<MemStore>,
-        parallel_count: usize,
-    ) {
+    async fn assert_cluster_manager(cluster_manager: &ClusterManager, parallel_count: usize) {
         let parallel_units = cluster_manager.list_active_streaming_parallel_units().await;
         assert_eq!(parallel_units.len(), parallel_count);
     }
diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs
index 803b9a8fbf3ed..e8a53f1db8359 100644
--- a/src/meta/src/manager/env.rs
+++ b/src/meta/src/manager/env.rs
@@ -27,24 +27,21 @@ use crate::manager::{
 use crate::model::ClusterId;
 #[cfg(any(test, feature = "test"))]
 use crate::storage::MemStore;
-use crate::storage::MetaStore;
+use crate::storage::{MetaStoreBoxExt, MetaStoreRef};
 use crate::MetaResult;
 
 /// [`MetaSrvEnv`] is the global environment in Meta service. The instance will be shared by all
 /// kind of managers inside Meta.
 #[derive(Clone)]
-pub struct MetaSrvEnv<S>
-where
-    S: MetaStore,
-{
+pub struct MetaSrvEnv {
     /// id generator manager.
-    id_gen_manager: IdGeneratorManagerRef<S>,
+    id_gen_manager: IdGeneratorManagerRef,
 
     /// meta store.
-    meta_store: Arc<S>,
+    meta_store: MetaStoreRef,
 
     /// notification manager.
-    notification_manager: NotificationManagerRef<S>,
+    notification_manager: NotificationManagerRef,
 
     /// stream client pool memorization.
     stream_client_pool: StreamClientPoolRef,
@@ -53,7 +50,7 @@ where
     idle_manager: IdleManagerRef,
 
     /// system param manager.
-    system_params_manager: SystemParamsManagerRef<S>,
+    system_params_manager: SystemParamsManagerRef,
 
     /// Unique identifier of the cluster.
     cluster_id: ClusterId,
@@ -203,14 +200,11 @@ impl MetaOpts {
     }
 }
 
-impl<S> MetaSrvEnv<S>
-where
-    S: MetaStore,
-{
+impl MetaSrvEnv {
     pub async fn new(
         opts: MetaOpts,
         init_system_params: SystemParams,
-        meta_store: Arc<S>,
+        meta_store: MetaStoreRef,
     ) -> MetaResult<Self> {
         // change to sync after refactor `IdGeneratorManager::new` sync.
         let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await);
@@ -218,7 +212,7 @@ where
         let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await);
         let idle_manager = Arc::new(IdleManager::new(opts.max_idle_ms));
         let (cluster_id, cluster_first_launch) =
-            if let Some(id) = ClusterId::from_meta_store(meta_store.deref()).await? {
+            if let Some(id) = ClusterId::from_meta_store(&meta_store).await? {
                 (id, false)
             } else {
                 (ClusterId::new(), true)
@@ -249,27 +243,27 @@ where
         })
     }
 
-    pub fn meta_store_ref(&self) -> Arc<S> {
+    pub fn meta_store_ref(&self) -> MetaStoreRef {
         self.meta_store.clone()
     }
 
-    pub fn meta_store(&self) -> &S {
-        self.meta_store.deref()
+    pub fn meta_store(&self) -> &MetaStoreRef {
+        &self.meta_store
     }
 
-    pub fn id_gen_manager_ref(&self) -> IdGeneratorManagerRef<S> {
+    pub fn id_gen_manager_ref(&self) -> IdGeneratorManagerRef {
         self.id_gen_manager.clone()
     }
 
-    pub fn id_gen_manager(&self) -> &IdGeneratorManager<S> {
+    pub fn id_gen_manager(&self) -> &IdGeneratorManager {
         self.id_gen_manager.deref()
     }
 
-    pub fn notification_manager_ref(&self) -> NotificationManagerRef<S> {
+    pub fn notification_manager_ref(&self) -> NotificationManagerRef {
         self.notification_manager.clone()
     }
 
-    pub fn notification_manager(&self) -> &NotificationManager<S> {
+    pub fn notification_manager(&self) -> &NotificationManager {
         self.notification_manager.deref()
     }
 
@@ -281,11 +275,11 @@ where
         self.idle_manager.deref()
     }
 
-    pub fn system_params_manager_ref(&self) -> SystemParamsManagerRef<S> {
+    pub fn system_params_manager_ref(&self) -> SystemParamsManagerRef {
         self.system_params_manager.clone()
     }
 
-    pub fn system_params_manager(&self) -> &SystemParamsManager<S> {
+    pub fn system_params_manager(&self) -> &SystemParamsManager {
         self.system_params_manager.deref()
     }
 
@@ -311,7 +305,7 @@ where
 }
 
 #[cfg(any(test, feature = "test"))]
-impl MetaSrvEnv<MemStore> {
+impl MetaSrvEnv {
     // Instance for test.
     pub async fn for_test() -> Self {
         Self::for_test_opts(MetaOpts::test(false).into()).await
@@ -319,7 +313,7 @@ impl MetaSrvEnv<MemStore> {
 
     pub async fn for_test_opts(opts: Arc<MetaOpts>) -> Self {
         // change to sync after refactor `IdGeneratorManager::new` sync.
-        let meta_store = Arc::new(MemStore::default());
+        let meta_store = MemStore::default().into_ref();
         let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await);
         let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await);
         let stream_client_pool = Arc::new(StreamClientPool::default());
diff --git a/src/meta/src/manager/id.rs b/src/meta/src/manager/id.rs
index 88abbe8a98e80..dbb2f53acf741 100644
--- a/src/meta/src/manager/id.rs
+++ b/src/meta/src/manager/id.rs
@@ -21,7 +21,7 @@ use tokio::sync::RwLock;
 
 use crate::manager::cluster::META_NODE_ID;
 use crate::model::MetadataModelResult;
-use crate::storage::{MetaStore, MetaStoreError, DEFAULT_COLUMN_FAMILY};
+use crate::storage::{MetaStore, MetaStoreError, MetaStoreRef, DEFAULT_COLUMN_FAMILY};
 
 pub const ID_PREALLOCATE_INTERVAL: u64 = 1000;
 
@@ -41,18 +41,15 @@ pub trait IdGenerator: Sync + Send + 'static {
 }
 
 /// [`StoredIdGenerator`] implements id generator using metastore.
-pub struct StoredIdGenerator<S> {
-    meta_store: Arc<S>,
+pub struct StoredIdGenerator {
+    meta_store: MetaStoreRef,
     category_gen_key: String,
     current_id: AtomicU64,
     next_allocate_id: RwLock<Id>,
 }
 
-impl<S> StoredIdGenerator<S>
-where
-    S: MetaStore,
-{
-    pub async fn new(meta_store: Arc<S>, category: &str, start: Option<Id>) -> Self {
+impl StoredIdGenerator {
+    pub async fn new(meta_store: MetaStoreRef, category: &str, start: Option<Id>) -> Self {
         let category_gen_key = format!("{}_id_next_generator", category);
         let res = meta_store
             .get_cf(DEFAULT_COLUMN_FAMILY, category_gen_key.as_bytes())
@@ -85,10 +82,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S> IdGenerator for StoredIdGenerator<S>
-where
-    S: MetaStore,
-{
+impl IdGenerator for StoredIdGenerator {
     async fn generate_interval(&self, interval: u64) -> MetadataModelResult<Id> {
         let id = self.current_id.fetch_add(interval, Ordering::Relaxed);
         let next_allocate_id = { *self.next_allocate_id.read().await };
@@ -144,34 +138,31 @@ pub mod IdCategory {
     pub const Connection: IdCategoryType = 17;
 }
 
-pub type IdGeneratorManagerRef<S> = Arc<IdGeneratorManager<S>>;
+pub type IdGeneratorManagerRef = Arc<IdGeneratorManager>;
 
 /// [`IdGeneratorManager`] manages id generators in all categories,
 /// which defined as [`IdCategory`] in [`meta.proto`].
-pub struct IdGeneratorManager<S> {
+pub struct IdGeneratorManager {
     #[cfg(test)]
-    test: Arc<StoredIdGenerator<S>>,
-    database: Arc<StoredIdGenerator<S>>,
-    schema: Arc<StoredIdGenerator<S>>,
-    table: Arc<StoredIdGenerator<S>>,
-    function: Arc<StoredIdGenerator<S>>,
-    worker: Arc<StoredIdGenerator<S>>,
-    fragment: Arc<StoredIdGenerator<S>>,
-    actor: Arc<StoredIdGenerator<S>>,
-    user: Arc<StoredIdGenerator<S>>,
-    backup: Arc<StoredIdGenerator<S>>,
-    hummock_ss_table_id: Arc<StoredIdGenerator<S>>,
-    hummock_compaction_task: Arc<StoredIdGenerator<S>>,
-    parallel_unit: Arc<StoredIdGenerator<S>>,
-    compaction_group: Arc<StoredIdGenerator<S>>,
-    connection: Arc<StoredIdGenerator<S>>,
+    test: Arc<StoredIdGenerator>,
+    database: Arc<StoredIdGenerator>,
+    schema: Arc<StoredIdGenerator>,
+    table: Arc<StoredIdGenerator>,
+    function: Arc<StoredIdGenerator>,
+    worker: Arc<StoredIdGenerator>,
+    fragment: Arc<StoredIdGenerator>,
+    actor: Arc<StoredIdGenerator>,
+    user: Arc<StoredIdGenerator>,
+    backup: Arc<StoredIdGenerator>,
+    hummock_ss_table_id: Arc<StoredIdGenerator>,
+    hummock_compaction_task: Arc<StoredIdGenerator>,
+    parallel_unit: Arc<StoredIdGenerator>,
+    compaction_group: Arc<StoredIdGenerator>,
+    connection: Arc<StoredIdGenerator>,
 }
 
-impl<S> IdGeneratorManager<S>
-where
-    S: MetaStore,
-{
-    pub async fn new(meta_store: Arc<S>) -> Self {
+impl IdGeneratorManager {
+    pub async fn new(meta_store: MetaStoreRef) -> Self {
         Self {
             #[cfg(test)]
             test: Arc::new(StoredIdGenerator::new(meta_store.clone(), "test", None).await),
@@ -227,7 +218,7 @@ where
         }
     }
 
-    const fn get<const C: IdCategoryType>(&self) -> &Arc<StoredIdGenerator<S>> {
+    const fn get<const C: IdCategoryType>(&self) -> &Arc<StoredIdGenerator> {
         match C {
             #[cfg(test)]
             IdCategory::Test => &self.test,
@@ -266,16 +257,14 @@ where
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
     use futures::future;
 
     use super::*;
-    use crate::storage::MemStore;
+    use crate::storage::{MemStore, MetaStoreBoxExt};
 
     #[tokio::test]
     async fn test_id_generator() -> MetadataModelResult<()> {
-        let meta_store = Arc::new(MemStore::default());
+        let meta_store = MemStore::default().into_ref();
         let id_generator = StoredIdGenerator::new(meta_store.clone(), "default", None).await;
         let ids = future::join_all((0..10000).map(|_i| {
             let id_generator = &id_generator;
@@ -335,7 +324,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_id_generator_manager() -> MetadataModelResult<()> {
-        let meta_store = Arc::new(MemStore::default());
+        let meta_store = MemStore::default().into_ref();
         let manager = IdGeneratorManager::new(meta_store.clone()).await;
         let ids = future::join_all((0..10000).map(|_i| {
             let manager = &manager;
diff --git a/src/meta/src/manager/notification.rs b/src/meta/src/manager/notification.rs
index 2f3cbec4f8a58..5e4172911ba70 100644
--- a/src/meta/src/manager/notification.rs
+++ b/src/meta/src/manager/notification.rs
@@ -30,11 +30,11 @@ use tonic::Status;
 
 use crate::manager::cluster::WorkerKey;
 use crate::model::{FragmentId, NotificationVersion as Version};
-use crate::storage::MetaStore;
+use crate::storage::MetaStoreRef;
 
 pub type MessageStatus = Status;
 pub type Notification = Result<SubscribeResponse, Status>;
-pub type NotificationManagerRef<S> = Arc<NotificationManager<S>>;
+pub type NotificationManagerRef = Arc<NotificationManager>;
 pub type NotificationVersion = u64;
 
 #[derive(Clone, Debug)]
@@ -72,20 +72,17 @@ struct Task {
 }
 
 /// [`NotificationManager`] is used to send notification to frontends and compute nodes.
-pub struct NotificationManager<S> {
+pub struct NotificationManager {
     core: Arc<Mutex<NotificationManagerCore>>,
     /// Sender used to add a notification into the waiting queue.
     task_tx: UnboundedSender<Task>,
     /// The current notification version.
     current_version: Mutex<Version>,
-    meta_store: Arc<S>,
+    meta_store: MetaStoreRef,
 }
 
-impl<S> NotificationManager<S>
-where
-    S: MetaStore,
-{
-    pub async fn new(meta_store: Arc<S>) -> Self {
+impl NotificationManager {
+    pub async fn new(meta_store: MetaStoreRef) -> Self {
         // notification waiting queue.
         let (task_tx, mut task_rx) = mpsc::unbounded_channel::<Task>();
         let core = Arc::new(Mutex::new(NotificationManagerCore::new()));
@@ -107,7 +104,7 @@ where
         Self {
             core: core_clone,
             task_tx,
-            current_version: Mutex::new(Version::new(&*meta_store).await),
+            current_version: Mutex::new(Version::new(&meta_store).await),
             meta_store,
         }
     }
@@ -143,7 +140,7 @@ where
         info: Info,
     ) -> NotificationVersion {
         let mut version_guard = self.current_version.lock().await;
-        version_guard.increase_version(&*self.meta_store).await;
+        version_guard.increase_version(&self.meta_store).await;
         let version = version_guard.version();
         self.notify(target, operation, info, Some(version));
         version
@@ -409,11 +406,11 @@ mod tests {
     use risingwave_pb::common::HostAddress;
 
     use super::*;
-    use crate::storage::MemStore;
+    use crate::storage::{MemStore, MetaStoreBoxExt};
 
     #[tokio::test]
     async fn test_multiple_subscribers_one_worker() {
-        let mgr = NotificationManager::new(MemStore::new().into()).await;
+        let mgr = NotificationManager::new(MemStore::new().into_ref()).await;
         let worker_key1 = WorkerKey(HostAddress {
             host: "a".to_string(),
             port: 1,
diff --git a/src/meta/src/manager/sink_coordination/manager.rs b/src/meta/src/manager/sink_coordination/manager.rs
index 39aac08b6c4ad..b38d70119e576 100644
--- a/src/meta/src/manager/sink_coordination/manager.rs
+++ b/src/meta/src/manager/sink_coordination/manager.rs
@@ -422,6 +422,8 @@ mod tests {
             columns: vec![],
             pk_indices: vec![],
             sink_type: SinkType::AppendOnly,
+            db_name: "test".into(),
+            sink_from_name: "test".into(),
         };
 
         let epoch1 = 233;
@@ -593,6 +595,8 @@ mod tests {
             columns: vec![],
             pk_indices: vec![],
             sink_type: SinkType::AppendOnly,
+            db_name: "test".into(),
+            sink_from_name: "test".into(),
         };
 
         let (manager, (_join_handle, _stop_tx)) = SinkCoordinatorManager::start_worker(None);
@@ -629,6 +633,8 @@ mod tests {
             columns: vec![],
             pk_indices: vec![],
             sink_type: SinkType::AppendOnly,
+            db_name: "test".into(),
+            sink_from_name: "test".into(),
         };
 
         let epoch = 233;
@@ -709,6 +715,8 @@ mod tests {
             columns: vec![],
             pk_indices: vec![],
             sink_type: SinkType::AppendOnly,
+            db_name: "test".into(),
+            sink_from_name: "test".into(),
         };
 
         let epoch = 233;
diff --git a/src/meta/src/manager/system_param/mod.rs b/src/meta/src/manager/system_param/mod.rs
index 6b07382a621f5..861234bdfe9fe 100644
--- a/src/meta/src/manager/system_param/mod.rs
+++ b/src/meta/src/manager/system_param/mod.rs
@@ -32,30 +32,30 @@ use tracing::info;
 use self::model::SystemParamsModel;
 use super::NotificationManagerRef;
 use crate::model::{ValTransaction, VarTransaction};
-use crate::storage::{MetaStore, Transaction};
+use crate::storage::{MetaStore, MetaStoreRef, Transaction};
 use crate::{MetaError, MetaResult};
 
-pub type SystemParamsManagerRef<S> = Arc<SystemParamsManager<S>>;
+pub type SystemParamsManagerRef = Arc<SystemParamsManager>;
 
-pub struct SystemParamsManager<S: MetaStore> {
-    meta_store: Arc<S>,
+pub struct SystemParamsManager {
+    meta_store: MetaStoreRef,
     // Notify workers and local subscribers of parameter change.
-    notification_manager: NotificationManagerRef<S>,
+    notification_manager: NotificationManagerRef,
     // Cached parameters.
     params: RwLock<SystemParams>,
 }
 
-impl<S: MetaStore> SystemParamsManager<S> {
+impl SystemParamsManager {
     /// Return error if `init_params` conflict with persisted system params.
     pub async fn new(
-        meta_store: Arc<S>,
-        notification_manager: NotificationManagerRef<S>,
+        meta_store: MetaStoreRef,
+        notification_manager: NotificationManagerRef,
         init_params: SystemParams,
         cluster_first_launch: bool,
     ) -> MetaResult<Self> {
         let params = if cluster_first_launch {
             init_params
-        } else if let Some(persisted) = SystemParams::get(meta_store.as_ref()).await? {
+        } else if let Some(persisted) = SystemParams::get(&meta_store).await? {
             merge_params(persisted, init_params)
         } else {
             return Err(MetaError::system_param(
@@ -109,10 +109,7 @@ impl<S: MetaStore> SystemParamsManager<S> {
 
     /// Flush the cached params to meta store.
     pub async fn flush_params(&self) -> MetaResult<()> {
-        Ok(
-            SystemParams::insert(self.params.read().await.deref(), self.meta_store.as_ref())
-                .await?,
-        )
+        Ok(SystemParams::insert(self.params.read().await.deref(), &self.meta_store).await?)
     }
 
     // Periodically sync params to worker nodes.
diff --git a/src/meta/src/model/barrier.rs b/src/meta/src/model/barrier.rs
index 0a603d7c3b100..e146dd47489ec 100644
--- a/src/meta/src/model/barrier.rs
+++ b/src/meta/src/model/barrier.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_pb::meta::PausedReason;
+
 use crate::barrier::TracedEpoch;
 
 /// `BarrierManagerState` defines the necessary state of `GlobalBarrierManager`.
@@ -21,15 +23,34 @@ pub struct BarrierManagerState {
     /// There's no need to persist this field. On recovery, we will restore this from the latest
     /// committed snapshot in `HummockManager`.
     in_flight_prev_epoch: TracedEpoch,
+
+    /// Whether the cluster is paused and the reason.
+    paused_reason: Option<PausedReason>,
 }
 
 impl BarrierManagerState {
-    pub fn new(in_flight_prev_epoch: TracedEpoch) -> Self {
+    pub fn new(in_flight_prev_epoch: TracedEpoch, paused_reason: Option<PausedReason>) -> Self {
         Self {
             in_flight_prev_epoch,
+            paused_reason,
         }
     }
 
+    pub fn paused_reason(&self) -> Option<PausedReason> {
+        self.paused_reason
+    }
+
+    pub fn set_paused_reason(&mut self, paused_reason: Option<PausedReason>) {
+        if self.paused_reason != paused_reason {
+            tracing::info!(current = ?self.paused_reason, new = ?paused_reason, "update paused state");
+            self.paused_reason = paused_reason;
+        }
+    }
+
+    pub fn in_flight_prev_epoch(&self) -> &TracedEpoch {
+        &self.in_flight_prev_epoch
+    }
+
     /// Returns the epoch pair for the next barrier, and updates the state.
     pub fn next_epoch_pair(&mut self) -> (TracedEpoch, TracedEpoch) {
         let prev_epoch = self.in_flight_prev_epoch.clone();
diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs
index 731a3d40de1e1..a11fe815609cd 100644
--- a/src/meta/src/rpc/ddl_controller.rs
+++ b/src/meta/src/rpc/ddl_controller.rs
@@ -40,7 +40,6 @@ use crate::manager::{
 };
 use crate::model::{StreamEnvironment, TableFragments};
 use crate::rpc::cloud_provider::AwsEc2Client;
-use crate::storage::MetaStore;
 use crate::stream::{
     validate_sink, ActorGraphBuildResult, ActorGraphBuilder, CompleteStreamFragmentGraph,
     CreateStreamingJobContext, GlobalStreamManagerRef, ReplaceTableContext, SourceManagerRef,
@@ -103,32 +102,28 @@ pub enum DdlCommand {
 }
 
 #[derive(Clone)]
-pub struct DdlController<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct DdlController {
+    env: MetaSrvEnv,
 
-    catalog_manager: CatalogManagerRef<S>,
-    stream_manager: GlobalStreamManagerRef<S>,
-    source_manager: SourceManagerRef<S>,
-    cluster_manager: ClusterManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
-    barrier_manager: BarrierManagerRef<S>,
+    catalog_manager: CatalogManagerRef,
+    stream_manager: GlobalStreamManagerRef,
+    source_manager: SourceManagerRef,
+    cluster_manager: ClusterManagerRef,
+    fragment_manager: FragmentManagerRef,
+    barrier_manager: BarrierManagerRef,
 
     aws_client: Arc<Option<AwsEc2Client>>,
     // The semaphore is used to limit the number of concurrent streaming job creation.
-    creating_streaming_job_permits: Arc<CreatingStreamingJobPermit<S>>,
+    creating_streaming_job_permits: Arc<CreatingStreamingJobPermit>,
 }
 
 #[derive(Clone)]
-pub struct CreatingStreamingJobPermit<S: MetaStore> {
+pub struct CreatingStreamingJobPermit {
     semaphore: Arc<Semaphore>,
-    _phantom: std::marker::PhantomData<S>,
 }
 
-impl<S> CreatingStreamingJobPermit<S>
-where
-    S: MetaStore,
-{
-    async fn new(env: &MetaSrvEnv<S>) -> Self {
+impl CreatingStreamingJobPermit {
+    async fn new(env: &MetaSrvEnv) -> Self {
         let mut permits = env
             .system_params_manager()
             .get_params()
@@ -177,25 +172,19 @@ where
             }
         });
 
-        Self {
-            semaphore,
-            _phantom: std::marker::PhantomData,
-        }
+        Self { semaphore }
     }
 }
 
-impl<S> DdlController<S>
-where
-    S: MetaStore,
-{
+impl DdlController {
     pub(crate) async fn new(
-        env: MetaSrvEnv<S>,
-        catalog_manager: CatalogManagerRef<S>,
-        stream_manager: GlobalStreamManagerRef<S>,
-        source_manager: SourceManagerRef<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
-        barrier_manager: BarrierManagerRef<S>,
+        env: MetaSrvEnv,
+        catalog_manager: CatalogManagerRef,
+        stream_manager: GlobalStreamManagerRef,
+        source_manager: SourceManagerRef,
+        cluster_manager: ClusterManagerRef,
+        fragment_manager: FragmentManagerRef,
+        barrier_manager: BarrierManagerRef,
         aws_client: Arc<Option<AwsEc2Client>>,
     ) -> Self {
         let creating_streaming_job_permits = Arc::new(CreatingStreamingJobPermit::new(&env).await);
@@ -758,7 +747,7 @@ where
         &self,
         source_id: Option<SourceId>,
         table_id: TableId,
-        fragment_manager: FragmentManagerRef<S>,
+        fragment_manager: FragmentManagerRef,
         drop_mode: DropMode,
     ) -> MetaResult<(
         NotificationVersion,
diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs
index f4b1649f68284..c19c06d9ab2cd 100644
--- a/src/meta/src/rpc/metrics.rs
+++ b/src/meta/src/rpc/metrics.rs
@@ -38,7 +38,6 @@ use tokio::task::JoinHandle;
 use crate::hummock::HummockManagerRef;
 use crate::manager::{CatalogManagerRef, ClusterManagerRef, FragmentManagerRef};
 use crate::rpc::server::ElectionClientRef;
-use crate::storage::MetaStore;
 
 #[derive(Clone)]
 pub struct MetaMetrics {
@@ -679,6 +678,11 @@ impl MetaMetrics {
             compaction_event_loop_iteration_latency,
         }
     }
+
+    #[cfg(test)]
+    pub fn for_test(registry: &Registry) -> Self {
+        Self::new(registry)
+    }
 }
 impl Default for MetaMetrics {
     fn default() -> Self {
@@ -686,8 +690,8 @@ impl Default for MetaMetrics {
     }
 }
 
-pub async fn start_worker_info_monitor<S: MetaStore>(
-    cluster_manager: ClusterManagerRef<S>,
+pub async fn start_worker_info_monitor(
+    cluster_manager: ClusterManagerRef,
     election_client: Option<ElectionClientRef>,
     interval: Duration,
     meta_metrics: Arc<MetaMetrics>,
@@ -734,11 +738,11 @@ pub async fn start_worker_info_monitor<S: MetaStore>(
     (join_handle, shutdown_tx)
 }
 
-pub async fn start_fragment_info_monitor<S: MetaStore>(
-    cluster_manager: ClusterManagerRef<S>,
-    catalog_manager: CatalogManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
-    hummock_manager: HummockManagerRef<S>,
+pub async fn start_fragment_info_monitor(
+    cluster_manager: ClusterManagerRef,
+    catalog_manager: CatalogManagerRef,
+    fragment_manager: FragmentManagerRef,
+    hummock_manager: HummockManagerRef,
     meta_metrics: Arc<MetaMetrics>,
 ) -> (JoinHandle<()>, Sender<()>) {
     const COLLECT_INTERVAL_SECONDS: u64 = 60;
diff --git a/src/meta/src/rpc/server.rs b/src/meta/src/rpc/server.rs
index b5a9dafa4c4ca..a21eb60eecffb 100644
--- a/src/meta/src/rpc/server.rs
+++ b/src/meta/src/rpc/server.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::net::SocketAddr;
-use std::ops::Deref;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -22,6 +21,7 @@ use etcd_client::ConnectOptions;
 use futures::future::join_all;
 use itertools::Itertools;
 use regex::Regex;
+use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
 use risingwave_common::telemetry::manager::TelemetryManager;
 use risingwave_common::telemetry::telemetry_env_enabled;
@@ -81,7 +81,10 @@ use crate::rpc::service::system_params_service::SystemParamsServiceImpl;
 use crate::rpc::service::telemetry_service::TelemetryInfoServiceImpl;
 use crate::rpc::service::user_service::UserServiceImpl;
 use crate::serving::ServingVnodeMapping;
-use crate::storage::{EtcdMetaStore, MemStore, MetaStore, WrappedEtcdClient as EtcdClient};
+use crate::storage::{
+    EtcdMetaStore, MemStore, MetaStore, MetaStoreBoxExt, MetaStoreRef,
+    WrappedEtcdClient as EtcdClient,
+};
 use crate::stream::{GlobalStreamManager, SourceManager};
 use crate::telemetry::{MetaReportCreator, MetaTelemetryInfoFetcher};
 use crate::{hummock, serving, MetaError, MetaResult};
@@ -141,7 +144,7 @@ pub async fn rpc_serve(
                 EtcdClient::connect(endpoints.clone(), Some(options.clone()), auth_enabled)
                     .await
                     .map_err(|e| anyhow::anyhow!("failed to connect etcd {}", e))?;
-            let meta_store = Arc::new(EtcdMetaStore::new(client));
+            let meta_store = EtcdMetaStore::new(client).into_ref();
 
             // `with_keep_alive` option will break the long connection in election client.
             let mut election_options = ConnectOptions::default();
@@ -170,7 +173,7 @@ pub async fn rpc_serve(
             )
         }
         MetaStoreBackend::Mem => {
-            let meta_store = Arc::new(MemStore::new());
+            let meta_store = MemStore::new().into_ref();
             rpc_serve_with_store(
                 meta_store,
                 None,
@@ -185,8 +188,8 @@ pub async fn rpc_serve(
 }
 
 #[expect(clippy::type_complexity)]
-pub fn rpc_serve_with_store<S: MetaStore>(
-    meta_store: Arc<S>,
+pub fn rpc_serve_with_store(
+    meta_store: MetaStoreRef,
     election_client: Option<ElectionClientRef>,
     address_info: AddressInfo,
     max_cluster_heartbeat_interval: Duration,
@@ -304,26 +307,33 @@ pub async fn start_service_as_election_follower(
         .layer(TracingExtractLayer::new())
         .add_service(MetaMemberServiceServer::new(meta_member_srv))
         .add_service(HealthServer::new(health_srv))
-        .serve_with_shutdown(address_info.listen_addr, async move {
-            tokio::select! {
-                // shutdown service if all services should be shut down
-                res = svc_shutdown_rx.changed() => {
-                    match res {
-                        Ok(_) => tracing::info!("Shutting down services"),
-                        Err(_) => tracing::error!("Service shutdown sender dropped")
-                    }
-                },
-                // shutdown service if follower becomes leader
-                res = follower_shutdown_rx => {
-                    match res {
-                        Ok(_) => tracing::info!("Shutting down follower services"),
-                        Err(_) => tracing::error!("Follower service shutdown sender dropped")
-                    }
-                },
-            }
-        })
-        .await
-        .unwrap();
+        .monitored_serve_with_shutdown(
+            address_info.listen_addr,
+            "grpc-meta-follower-service",
+            TcpConfig {
+                tcp_nodelay: true,
+                keepalive_duration: None,
+            },
+            async move {
+                tokio::select! {
+                    // shutdown service if all services should be shut down
+                    res = svc_shutdown_rx.changed() => {
+                        match res {
+                            Ok(_) => tracing::info!("Shutting down services"),
+                            Err(_) => tracing::error!("Service shutdown sender dropped")
+                        }
+                    },
+                    // shutdown service if follower becomes leader
+                    res = follower_shutdown_rx => {
+                        match res {
+                            Ok(_) => tracing::info!("Shutting down follower services"),
+                            Err(_) => tracing::error!("Follower service shutdown sender dropped")
+                        }
+                    },
+                }
+            },
+        )
+        .await;
 }
 
 /// Starts all services needed for the meta leader node
@@ -332,8 +342,8 @@ pub async fn start_service_as_election_follower(
 ///
 /// ## Returns
 /// Returns an error if the service initialization failed
-pub async fn start_service_as_election_leader<S: MetaStore>(
-    meta_store: Arc<S>,
+pub async fn start_service_as_election_leader(
+    meta_store: MetaStoreRef,
     address_info: AddressInfo,
     max_cluster_heartbeat_interval: Duration,
     opts: MetaOpts,
@@ -343,7 +353,7 @@ pub async fn start_service_as_election_leader<S: MetaStore>(
 ) -> MetaResult<()> {
     tracing::info!("Defining leader services");
     let prometheus_endpoint = opts.prometheus_endpoint.clone();
-    let env = MetaSrvEnv::<S>::new(opts, init_system_params, meta_store.clone()).await?;
+    let env = MetaSrvEnv::new(opts, init_system_params, meta_store.clone()).await?;
     let fragment_manager = Arc::new(FragmentManager::new(env.clone()).await.unwrap());
 
     let system_params_manager = env.system_params_manager_ref();
@@ -512,7 +522,7 @@ pub async fn start_service_as_election_leader<S: MetaStore>(
         aws_cli = Some(cli);
     }
 
-    let ddl_srv = DdlServiceImpl::<S>::new(
+    let ddl_srv = DdlServiceImpl::new(
         env.clone(),
         aws_cli.clone(),
         catalog_manager.clone(),
@@ -525,19 +535,19 @@ pub async fn start_service_as_election_leader<S: MetaStore>(
     )
     .await;
 
-    let user_srv = UserServiceImpl::<S>::new(env.clone(), catalog_manager.clone());
+    let user_srv = UserServiceImpl::new(env.clone(), catalog_manager.clone());
 
-    let scale_srv = ScaleServiceImpl::<S>::new(
-        barrier_scheduler.clone(),
+    let scale_srv = ScaleServiceImpl::new(
         fragment_manager.clone(),
         cluster_manager.clone(),
         source_manager,
         catalog_manager.clone(),
         stream_manager.clone(),
+        barrier_manager.clone(),
     );
 
-    let cluster_srv = ClusterServiceImpl::<S>::new(cluster_manager.clone());
-    let stream_srv = StreamServiceImpl::<S>::new(
+    let cluster_srv = ClusterServiceImpl::new(cluster_manager.clone());
+    let stream_srv = StreamServiceImpl::new(
         env.clone(),
         barrier_scheduler.clone(),
         stream_manager.clone(),
@@ -565,7 +575,7 @@ pub async fn start_service_as_election_leader<S: MetaStore>(
     let system_params_srv = SystemParamsServiceImpl::new(system_params_manager.clone());
     let serving_srv =
         ServingServiceImpl::new(serving_vnode_mapping.clone(), fragment_manager.clone());
-    let cloud_srv = CloudServiceImpl::<S>::new(catalog_manager.clone(), aws_cli);
+    let cloud_srv = CloudServiceImpl::new(catalog_manager.clone(), aws_cli);
 
     if let Some(prometheus_addr) = address_info.prometheus_addr {
         MetricsManager::boot_metrics_service(prometheus_addr.to_string())
@@ -689,9 +699,7 @@ pub async fn start_service_as_election_leader<S: MetaStore>(
     // Persist params before starting services so that invalid params that cause meta node
     // to crash will not be persisted.
     system_params_manager.flush_params().await?;
-    env.cluster_id()
-        .put_at_meta_store(meta_store.deref())
-        .await?;
+    env.cluster_id().put_at_meta_store(&meta_store).await?;
 
     tracing::info!("Assigned cluster id {:?}", *env.cluster_id());
     tracing::info!("Starting meta services");
@@ -717,22 +725,29 @@ pub async fn start_service_as_election_leader<S: MetaStore>(
         .add_service(ServingServiceServer::new(serving_srv))
         .add_service(CloudServiceServer::new(cloud_srv))
         .add_service(SinkCoordinationServiceServer::new(sink_coordination_srv))
-        .serve_with_shutdown(address_info.listen_addr, async move {
-            tokio::select! {
-                res = svc_shutdown_rx.changed() => {
-                    match res {
-                        Ok(_) => tracing::info!("Shutting down services"),
-                        Err(_) => tracing::error!("Service shutdown receiver dropped")
-                    }
-                    shutdown_all.await;
-                },
-                _ = idle_recv => {
-                    shutdown_all.await;
-                },
-            }
-        })
-        .await
-        .unwrap();
+        .monitored_serve_with_shutdown(
+            address_info.listen_addr,
+            "grpc-meta-leader-service",
+            TcpConfig {
+                tcp_nodelay: true,
+                keepalive_duration: None,
+            },
+            async move {
+                tokio::select! {
+                    res = svc_shutdown_rx.changed() => {
+                        match res {
+                            Ok(_) => tracing::info!("Shutting down services"),
+                            Err(_) => tracing::error!("Service shutdown receiver dropped")
+                        }
+                        shutdown_all.await;
+                    },
+                    _ = idle_recv => {
+                        shutdown_all.await;
+                    },
+                }
+            },
+        )
+        .await;
 
     #[cfg(not(madsim))]
     if let Some(dashboard_task) = dashboard_task {
diff --git a/src/meta/src/rpc/service/backup_service.rs b/src/meta/src/rpc/service/backup_service.rs
index b7dfe55c8c99e..22897d8bb770e 100644
--- a/src/meta/src/rpc/service/backup_service.rs
+++ b/src/meta/src/rpc/service/backup_service.rs
@@ -23,26 +23,19 @@ use risingwave_pb::backup_service::{
 use tonic::{Request, Response, Status};
 
 use crate::backup_restore::BackupManagerRef;
-use crate::storage::MetaStore;
 
-pub struct BackupServiceImpl<S>
-where
-    S: MetaStore,
-{
-    backup_manager: BackupManagerRef<S>,
+pub struct BackupServiceImpl {
+    backup_manager: BackupManagerRef,
 }
 
-impl<S: MetaStore> BackupServiceImpl<S> {
-    pub fn new(backup_manager: BackupManagerRef<S>) -> Self {
+impl BackupServiceImpl {
+    pub fn new(backup_manager: BackupManagerRef) -> Self {
         Self { backup_manager }
     }
 }
 
 #[async_trait::async_trait]
-impl<S> BackupService for BackupServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl BackupService for BackupServiceImpl {
     async fn backup_meta(
         &self,
         _request: Request<BackupMetaRequest>,
diff --git a/src/meta/src/rpc/service/cloud_service.rs b/src/meta/src/rpc/service/cloud_service.rs
index 435bfd830affb..141abd9a0aab9 100644
--- a/src/meta/src/rpc/service/cloud_service.rs
+++ b/src/meta/src/rpc/service/cloud_service.rs
@@ -31,18 +31,14 @@ use tonic::{Request, Response, Status};
 
 use crate::manager::CatalogManagerRef;
 use crate::rpc::cloud_provider::AwsEc2Client;
-use crate::storage::MetaStore;
 
-pub struct CloudServiceImpl<S>
-where
-    S: MetaStore,
-{
-    catalog_manager: CatalogManagerRef<S>,
+pub struct CloudServiceImpl {
+    catalog_manager: CatalogManagerRef,
     aws_client: Option<AwsEc2Client>,
 }
 
-impl<S: MetaStore> CloudServiceImpl<S> {
-    pub fn new(catalog_manager: CatalogManagerRef<S>, aws_client: Option<AwsEc2Client>) -> Self {
+impl CloudServiceImpl {
+    pub fn new(catalog_manager: CatalogManagerRef, aws_client: Option<AwsEc2Client>) -> Self {
         Self {
             catalog_manager,
             aws_client,
@@ -65,10 +61,7 @@ fn new_rwc_validate_fail_response(
 }
 
 #[async_trait]
-impl<S> CloudService for CloudServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl CloudService for CloudServiceImpl {
     async fn rw_cloud_validate_source(
         &self,
         request: Request<RwCloudValidateSourceRequest>,
diff --git a/src/meta/src/rpc/service/cluster_service.rs b/src/meta/src/rpc/service/cluster_service.rs
index 7337ca1ca0257..1d6f87b5f9d01 100644
--- a/src/meta/src/rpc/service/cluster_service.rs
+++ b/src/meta/src/rpc/service/cluster_service.rs
@@ -23,28 +23,21 @@ use risingwave_pb::meta::{
 use tonic::{Request, Response, Status};
 
 use crate::manager::ClusterManagerRef;
-use crate::storage::MetaStore;
 use crate::MetaError;
 
 #[derive(Clone)]
-pub struct ClusterServiceImpl<S: MetaStore> {
-    cluster_manager: ClusterManagerRef<S>,
+pub struct ClusterServiceImpl {
+    cluster_manager: ClusterManagerRef,
 }
 
-impl<S> ClusterServiceImpl<S>
-where
-    S: MetaStore,
-{
-    pub fn new(cluster_manager: ClusterManagerRef<S>) -> Self {
+impl ClusterServiceImpl {
+    pub fn new(cluster_manager: ClusterManagerRef) -> Self {
         ClusterServiceImpl { cluster_manager }
     }
 }
 
 #[async_trait::async_trait]
-impl<S> ClusterService for ClusterServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl ClusterService for ClusterServiceImpl {
     async fn add_worker_node(
         &self,
         request: Request<AddWorkerNodeRequest>,
diff --git a/src/meta/src/rpc/service/ddl_service.rs b/src/meta/src/rpc/service/ddl_service.rs
index 186597647a7ab..a744ba910198d 100644
--- a/src/meta/src/rpc/service/ddl_service.rs
+++ b/src/meta/src/rpc/service/ddl_service.rs
@@ -40,34 +40,30 @@ use crate::manager::{
 };
 use crate::rpc::cloud_provider::AwsEc2Client;
 use crate::rpc::ddl_controller::{DdlCommand, DdlController, DropMode, StreamingJobId};
-use crate::storage::MetaStore;
 use crate::stream::{GlobalStreamManagerRef, SourceManagerRef};
 use crate::{MetaError, MetaResult};
 
 #[derive(Clone)]
-pub struct DdlServiceImpl<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct DdlServiceImpl {
+    env: MetaSrvEnv,
 
-    catalog_manager: CatalogManagerRef<S>,
+    catalog_manager: CatalogManagerRef,
     sink_manager: SinkCoordinatorManager,
-    ddl_controller: DdlController<S>,
+    ddl_controller: DdlController,
     aws_client: Arc<Option<AwsEc2Client>>,
 }
 
-impl<S> DdlServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl DdlServiceImpl {
     #[allow(clippy::too_many_arguments)]
     pub async fn new(
-        env: MetaSrvEnv<S>,
+        env: MetaSrvEnv,
         aws_client: Option<AwsEc2Client>,
-        catalog_manager: CatalogManagerRef<S>,
-        stream_manager: GlobalStreamManagerRef<S>,
-        source_manager: SourceManagerRef<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
-        barrier_manager: BarrierManagerRef<S>,
+        catalog_manager: CatalogManagerRef,
+        stream_manager: GlobalStreamManagerRef,
+        source_manager: SourceManagerRef,
+        cluster_manager: ClusterManagerRef,
+        fragment_manager: FragmentManagerRef,
+        barrier_manager: BarrierManagerRef,
         sink_manager: SinkCoordinatorManager,
     ) -> Self {
         let aws_cli_ref = Arc::new(aws_client);
@@ -93,10 +89,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S> DdlService for DdlServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl DdlService for DdlServiceImpl {
     async fn create_database(
         &self,
         request: Request<CreateDatabaseRequest>,
@@ -737,10 +730,7 @@ where
     }
 }
 
-impl<S> DdlServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl DdlServiceImpl {
     async fn gen_unique_id<const C: IdCategoryType>(&self) -> MetaResult<u32> {
         let id = self.env.id_gen_manager().generate::<C>().await? as u32;
         Ok(id)
diff --git a/src/meta/src/rpc/service/heartbeat_service.rs b/src/meta/src/rpc/service/heartbeat_service.rs
index d10d5994c577c..7c51b39346894 100644
--- a/src/meta/src/rpc/service/heartbeat_service.rs
+++ b/src/meta/src/rpc/service/heartbeat_service.rs
@@ -18,30 +18,20 @@ use risingwave_pb::meta::{HeartbeatRequest, HeartbeatResponse};
 use tonic::{Request, Response, Status};
 
 use crate::manager::ClusterManagerRef;
-use crate::storage::MetaStore;
 
 #[derive(Clone)]
-pub struct HeartbeatServiceImpl<S>
-where
-    S: MetaStore,
-{
-    cluster_manager: ClusterManagerRef<S>,
+pub struct HeartbeatServiceImpl {
+    cluster_manager: ClusterManagerRef,
 }
 
-impl<S> HeartbeatServiceImpl<S>
-where
-    S: MetaStore,
-{
-    pub fn new(cluster_manager: ClusterManagerRef<S>) -> Self {
+impl HeartbeatServiceImpl {
+    pub fn new(cluster_manager: ClusterManagerRef) -> Self {
         HeartbeatServiceImpl { cluster_manager }
     }
 }
 
 #[async_trait::async_trait]
-impl<S> HeartbeatService for HeartbeatServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl HeartbeatService for HeartbeatServiceImpl {
     #[cfg_attr(coverage, no_coverage)]
     async fn heartbeat(
         &self,
diff --git a/src/meta/src/rpc/service/hummock_service.rs b/src/meta/src/rpc/service/hummock_service.rs
index d3a0ec0d5f1c7..58310cc6eb110 100644
--- a/src/meta/src/rpc/service/hummock_service.rs
+++ b/src/meta/src/rpc/service/hummock_service.rs
@@ -28,24 +28,17 @@ use crate::hummock::compaction::ManualCompactionOption;
 use crate::hummock::{HummockManagerRef, VacuumManagerRef};
 use crate::manager::FragmentManagerRef;
 use crate::rpc::service::RwReceiverStream;
-use crate::storage::MetaStore;
-pub struct HummockServiceImpl<S>
-where
-    S: MetaStore,
-{
-    hummock_manager: HummockManagerRef<S>,
-    vacuum_manager: VacuumManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
+pub struct HummockServiceImpl {
+    hummock_manager: HummockManagerRef,
+    vacuum_manager: VacuumManagerRef,
+    fragment_manager: FragmentManagerRef,
 }
 
-impl<S> HummockServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl HummockServiceImpl {
     pub fn new(
-        hummock_manager: HummockManagerRef<S>,
-        vacuum_trigger: VacuumManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
+        hummock_manager: HummockManagerRef,
+        vacuum_trigger: VacuumManagerRef,
+        fragment_manager: FragmentManagerRef,
     ) -> Self {
         HummockServiceImpl {
             hummock_manager,
@@ -56,10 +49,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S> HummockManagerService for HummockServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl HummockManagerService for HummockServiceImpl {
     type SubscribeCompactionEventStream = RwReceiverStream<SubscribeCompactionEventResponse>;
 
     async fn unpin_version_before(
diff --git a/src/meta/src/rpc/service/notification_service.rs b/src/meta/src/rpc/service/notification_service.rs
index 2c47c215677b6..0fcbfe4929ec6 100644
--- a/src/meta/src/rpc/service/notification_service.rs
+++ b/src/meta/src/rpc/service/notification_service.rs
@@ -35,30 +35,26 @@ use crate::manager::{
     NotificationVersion, WorkerKey,
 };
 use crate::serving::ServingVnodeMappingRef;
-use crate::storage::MetaStore;
 
-pub struct NotificationServiceImpl<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct NotificationServiceImpl {
+    env: MetaSrvEnv,
 
-    catalog_manager: CatalogManagerRef<S>,
-    cluster_manager: ClusterManagerRef<S>,
-    hummock_manager: HummockManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
-    backup_manager: BackupManagerRef<S>,
+    catalog_manager: CatalogManagerRef,
+    cluster_manager: ClusterManagerRef,
+    hummock_manager: HummockManagerRef,
+    fragment_manager: FragmentManagerRef,
+    backup_manager: BackupManagerRef,
     serving_vnode_mapping: ServingVnodeMappingRef,
 }
 
-impl<S> NotificationServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl NotificationServiceImpl {
     pub fn new(
-        env: MetaSrvEnv<S>,
-        catalog_manager: CatalogManagerRef<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        hummock_manager: HummockManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
-        backup_manager: BackupManagerRef<S>,
+        env: MetaSrvEnv,
+        catalog_manager: CatalogManagerRef,
+        cluster_manager: ClusterManagerRef,
+        hummock_manager: HummockManagerRef,
+        fragment_manager: FragmentManagerRef,
+        backup_manager: BackupManagerRef,
         serving_vnode_mapping: ServingVnodeMappingRef,
     ) -> Self {
         Self {
@@ -209,10 +205,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S> NotificationService for NotificationServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl NotificationService for NotificationServiceImpl {
     type SubscribeStream = UnboundedReceiverStream<Notification>;
 
     #[cfg_attr(coverage, no_coverage)]
diff --git a/src/meta/src/rpc/service/scale_service.rs b/src/meta/src/rpc/service/scale_service.rs
index a770a1d726043..f231ea5f4955d 100644
--- a/src/meta/src/rpc/service/scale_service.rs
+++ b/src/meta/src/rpc/service/scale_service.rs
@@ -16,71 +16,49 @@ use risingwave_pb::common::WorkerType;
 use risingwave_pb::meta::scale_service_server::ScaleService;
 use risingwave_pb::meta::{
     GetClusterInfoRequest, GetClusterInfoResponse, GetReschedulePlanRequest,
-    GetReschedulePlanResponse, PauseRequest, PauseResponse, Reschedule, RescheduleRequest,
-    RescheduleResponse, ResumeRequest, ResumeResponse,
+    GetReschedulePlanResponse, Reschedule, RescheduleRequest, RescheduleResponse,
 };
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
 use tonic::{Request, Response, Status};
 
-use crate::barrier::{BarrierScheduler, Command};
+use crate::barrier::BarrierManagerRef;
 use crate::manager::{CatalogManagerRef, ClusterManagerRef, FragmentManagerRef};
 use crate::model::MetadataModel;
-use crate::storage::MetaStore;
 use crate::stream::{
     GlobalStreamManagerRef, ParallelUnitReschedule, RescheduleOptions, SourceManagerRef,
 };
 
-pub struct ScaleServiceImpl<S: MetaStore> {
-    barrier_scheduler: BarrierScheduler<S>,
-    fragment_manager: FragmentManagerRef<S>,
-    cluster_manager: ClusterManagerRef<S>,
-    source_manager: SourceManagerRef<S>,
-    catalog_manager: CatalogManagerRef<S>,
-    stream_manager: GlobalStreamManagerRef<S>,
+pub struct ScaleServiceImpl {
+    fragment_manager: FragmentManagerRef,
+    cluster_manager: ClusterManagerRef,
+    source_manager: SourceManagerRef,
+    catalog_manager: CatalogManagerRef,
+    stream_manager: GlobalStreamManagerRef,
+    barrier_manager: BarrierManagerRef,
 }
 
-impl<S> ScaleServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl ScaleServiceImpl {
     pub fn new(
-        barrier_scheduler: BarrierScheduler<S>,
-        fragment_manager: FragmentManagerRef<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        source_manager: SourceManagerRef<S>,
-        catalog_manager: CatalogManagerRef<S>,
-        stream_manager: GlobalStreamManagerRef<S>,
+        fragment_manager: FragmentManagerRef,
+        cluster_manager: ClusterManagerRef,
+        source_manager: SourceManagerRef,
+        catalog_manager: CatalogManagerRef,
+        stream_manager: GlobalStreamManagerRef,
+        barrier_manager: BarrierManagerRef,
     ) -> Self {
         Self {
-            barrier_scheduler,
             fragment_manager,
             cluster_manager,
             source_manager,
             catalog_manager,
             stream_manager,
+            barrier_manager,
         }
     }
 }
 
 #[async_trait::async_trait]
-impl<S> ScaleService for ScaleServiceImpl<S>
-where
-    S: MetaStore,
-{
-    #[cfg_attr(coverage, no_coverage)]
-    async fn pause(&self, _: Request<PauseRequest>) -> Result<Response<PauseResponse>, Status> {
-        self.barrier_scheduler.run_command(Command::pause()).await?;
-        Ok(Response::new(PauseResponse {}))
-    }
-
-    #[cfg_attr(coverage, no_coverage)]
-    async fn resume(&self, _: Request<ResumeRequest>) -> Result<Response<ResumeResponse>, Status> {
-        self.barrier_scheduler
-            .run_command(Command::resume())
-            .await?;
-        Ok(Response::new(ResumeResponse {}))
-    }
-
+impl ScaleService for ScaleServiceImpl {
     #[cfg_attr(coverage, no_coverage)]
     async fn get_cluster_info(
         &self,
@@ -137,6 +115,12 @@ where
         &self,
         request: Request<RescheduleRequest>,
     ) -> Result<Response<RescheduleResponse>, Status> {
+        if !self.barrier_manager.is_running().await {
+            return Err(Status::unavailable(
+                "Rescheduling is unavailable for now. Likely the cluster is starting or recovering.",
+            ));
+        }
+
         let RescheduleRequest {
             reschedules,
             revision,
@@ -197,6 +181,12 @@ where
     ) -> Result<Response<GetReschedulePlanResponse>, Status> {
         let req = request.into_inner();
 
+        if !self.barrier_manager.is_running().await {
+            return Err(Status::unavailable(
+                "Rescheduling is unavailable for now. Likely the cluster is starting or recovering.",
+            ));
+        }
+
         let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await;
 
         let current_revision = self.fragment_manager.get_revision().await;
diff --git a/src/meta/src/rpc/service/serving_service.rs b/src/meta/src/rpc/service/serving_service.rs
index c19aad6d9f225..b3270828f9bf5 100644
--- a/src/meta/src/rpc/service/serving_service.rs
+++ b/src/meta/src/rpc/service/serving_service.rs
@@ -21,20 +21,16 @@ use tonic::{Request, Response, Status};
 
 use crate::manager::FragmentManagerRef;
 use crate::serving::ServingVnodeMappingRef;
-use crate::storage::MetaStore;
 
-pub struct ServingServiceImpl<S: MetaStore> {
+pub struct ServingServiceImpl {
     serving_vnode_mapping: ServingVnodeMappingRef,
-    fragment_manager: FragmentManagerRef<S>,
+    fragment_manager: FragmentManagerRef,
 }
 
-impl<S> ServingServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl ServingServiceImpl {
     pub fn new(
         serving_vnode_mapping: ServingVnodeMappingRef,
-        fragment_manager: FragmentManagerRef<S>,
+        fragment_manager: FragmentManagerRef,
     ) -> Self {
         Self {
             serving_vnode_mapping,
@@ -44,10 +40,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S> ServingService for ServingServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl ServingService for ServingServiceImpl {
     async fn get_serving_vnode_mappings(
         &self,
         _request: Request<GetServingVnodeMappingsRequest>,
diff --git a/src/meta/src/rpc/service/stream_service.rs b/src/meta/src/rpc/service/stream_service.rs
index c9a42b33113f0..b2ed1ec916b08 100644
--- a/src/meta/src/rpc/service/stream_service.rs
+++ b/src/meta/src/rpc/service/stream_service.rs
@@ -24,35 +24,28 @@ use risingwave_pb::meta::stream_manager_service_server::StreamManagerService;
 use risingwave_pb::meta::*;
 use tonic::{Request, Response, Status};
 
-use crate::barrier::BarrierScheduler;
+use crate::barrier::{BarrierScheduler, Command};
 use crate::manager::{CatalogManagerRef, FragmentManagerRef, MetaSrvEnv};
-use crate::storage::MetaStore;
 use crate::stream::GlobalStreamManagerRef;
 
 pub type TonicResponse<T> = Result<Response<T>, Status>;
 
 #[derive(Clone)]
-pub struct StreamServiceImpl<S>
-where
-    S: MetaStore,
-{
-    env: MetaSrvEnv<S>,
-    barrier_scheduler: BarrierScheduler<S>,
-    stream_manager: GlobalStreamManagerRef<S>,
-    catalog_manager: CatalogManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
+pub struct StreamServiceImpl {
+    env: MetaSrvEnv,
+    barrier_scheduler: BarrierScheduler,
+    stream_manager: GlobalStreamManagerRef,
+    catalog_manager: CatalogManagerRef,
+    fragment_manager: FragmentManagerRef,
 }
 
-impl<S> StreamServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl StreamServiceImpl {
     pub fn new(
-        env: MetaSrvEnv<S>,
-        barrier_scheduler: BarrierScheduler<S>,
-        stream_manager: GlobalStreamManagerRef<S>,
-        catalog_manager: CatalogManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
+        env: MetaSrvEnv,
+        barrier_scheduler: BarrierScheduler,
+        stream_manager: GlobalStreamManagerRef,
+        catalog_manager: CatalogManagerRef,
+        fragment_manager: FragmentManagerRef,
     ) -> Self {
         StreamServiceImpl {
             env,
@@ -65,10 +58,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S> StreamManagerService for StreamServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl StreamManagerService for StreamServiceImpl {
     #[cfg_attr(coverage, no_coverage)]
     async fn flush(&self, request: Request<FlushRequest>) -> TonicResponse<FlushResponse> {
         self.env.idle_manager().record_activity();
@@ -81,6 +71,30 @@ where
         }))
     }
 
+    #[cfg_attr(coverage, no_coverage)]
+    async fn pause(&self, _: Request<PauseRequest>) -> Result<Response<PauseResponse>, Status> {
+        let i = self
+            .barrier_scheduler
+            .run_command(Command::pause(PausedReason::Manual))
+            .await?;
+        Ok(Response::new(PauseResponse {
+            prev: i.prev_paused_reason.map(Into::into),
+            curr: i.curr_paused_reason.map(Into::into),
+        }))
+    }
+
+    #[cfg_attr(coverage, no_coverage)]
+    async fn resume(&self, _: Request<ResumeRequest>) -> Result<Response<ResumeResponse>, Status> {
+        let i = self
+            .barrier_scheduler
+            .run_command(Command::resume(PausedReason::Manual))
+            .await?;
+        Ok(Response::new(ResumeResponse {
+            prev: i.prev_paused_reason.map(Into::into),
+            curr: i.curr_paused_reason.map(Into::into),
+        }))
+    }
+
     async fn cancel_creating_jobs(
         &self,
         request: Request<CancelCreatingJobsRequest>,
diff --git a/src/meta/src/rpc/service/system_params_service.rs b/src/meta/src/rpc/service/system_params_service.rs
index 4e93bda21a9d1..114c9aa917a68 100644
--- a/src/meta/src/rpc/service/system_params_service.rs
+++ b/src/meta/src/rpc/service/system_params_service.rs
@@ -20,17 +20,13 @@ use risingwave_pb::meta::{
 use tonic::{Request, Response, Status};
 
 use crate::manager::SystemParamsManagerRef;
-use crate::storage::MetaStore;
 
-pub struct SystemParamsServiceImpl<S>
-where
-    S: MetaStore,
-{
-    system_params_manager: SystemParamsManagerRef<S>,
+pub struct SystemParamsServiceImpl {
+    system_params_manager: SystemParamsManagerRef,
 }
 
-impl<S: MetaStore> SystemParamsServiceImpl<S> {
-    pub fn new(system_params_manager: SystemParamsManagerRef<S>) -> Self {
+impl SystemParamsServiceImpl {
+    pub fn new(system_params_manager: SystemParamsManagerRef) -> Self {
         Self {
             system_params_manager,
         }
@@ -38,10 +34,7 @@ impl<S: MetaStore> SystemParamsServiceImpl<S> {
 }
 
 #[async_trait]
-impl<S> SystemParamsService for SystemParamsServiceImpl<S>
-where
-    S: MetaStore,
-{
+impl SystemParamsService for SystemParamsServiceImpl {
     async fn get_system_params(
         &self,
         _request: Request<GetSystemParamsRequest>,
diff --git a/src/meta/src/rpc/service/telemetry_service.rs b/src/meta/src/rpc/service/telemetry_service.rs
index 53ffa52e17b55..b1a9cdec3ef34 100644
--- a/src/meta/src/rpc/service/telemetry_service.rs
+++ b/src/meta/src/rpc/service/telemetry_service.rs
@@ -12,27 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::ops::Deref;
-use std::sync::Arc;
-
 use risingwave_pb::meta::telemetry_info_service_server::TelemetryInfoService;
 use risingwave_pb::meta::{GetTelemetryInfoRequest, TelemetryInfoResponse};
 use tonic::{Request, Response, Status};
 
 use crate::model::ClusterId;
-use crate::storage::MetaStore;
+use crate::storage::MetaStoreRef;
 
-pub struct TelemetryInfoServiceImpl<S: MetaStore> {
-    meta_store: Arc<S>,
+pub struct TelemetryInfoServiceImpl {
+    meta_store: MetaStoreRef,
 }
 
-impl<S: MetaStore> TelemetryInfoServiceImpl<S> {
-    pub fn new(meta_store: Arc<S>) -> Self {
+impl TelemetryInfoServiceImpl {
+    pub fn new(meta_store: MetaStoreRef) -> Self {
         Self { meta_store }
     }
 
     async fn get_tracking_id(&self) -> Option<ClusterId> {
-        ClusterId::from_meta_store(self.meta_store.deref())
+        ClusterId::from_meta_store(&self.meta_store)
             .await
             .ok()
             .flatten()
@@ -40,7 +37,7 @@ impl<S: MetaStore> TelemetryInfoServiceImpl<S> {
 }
 
 #[async_trait::async_trait]
-impl<S: MetaStore> TelemetryInfoService for TelemetryInfoServiceImpl<S> {
+impl TelemetryInfoService for TelemetryInfoServiceImpl {
     async fn get_telemetry_info(
         &self,
         _request: Request<GetTelemetryInfoRequest>,
diff --git a/src/meta/src/rpc/service/user_service.rs b/src/meta/src/rpc/service/user_service.rs
index baaceea65d3c6..e1b7cc27092d5 100644
--- a/src/meta/src/rpc/service/user_service.rs
+++ b/src/meta/src/rpc/service/user_service.rs
@@ -24,20 +24,16 @@ use risingwave_pb::user::{
 use tonic::{Request, Response, Status};
 
 use crate::manager::{CatalogManagerRef, IdCategory, MetaSrvEnv};
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
-pub struct UserServiceImpl<S: MetaStore> {
-    env: MetaSrvEnv<S>,
+pub struct UserServiceImpl {
+    env: MetaSrvEnv,
 
-    catalog_manager: CatalogManagerRef<S>,
+    catalog_manager: CatalogManagerRef,
 }
 
-impl<S> UserServiceImpl<S>
-where
-    S: MetaStore,
-{
-    pub fn new(env: MetaSrvEnv<S>, catalog_manager: CatalogManagerRef<S>) -> Self {
+impl UserServiceImpl {
+    pub fn new(env: MetaSrvEnv, catalog_manager: CatalogManagerRef) -> Self {
         Self {
             env,
             catalog_manager,
@@ -110,7 +106,7 @@ where
 }
 
 #[async_trait::async_trait]
-impl<S: MetaStore> UserService for UserServiceImpl<S> {
+impl UserService for UserServiceImpl {
     #[cfg_attr(coverage, no_coverage)]
     async fn create_user(
         &self,
diff --git a/src/meta/src/serving/mod.rs b/src/meta/src/serving/mod.rs
index 4da9af3503d94..f6d1a5b1aa714 100644
--- a/src/meta/src/serving/mod.rs
+++ b/src/meta/src/serving/mod.rs
@@ -28,7 +28,6 @@ use crate::manager::{
     ClusterManagerRef, FragmentManagerRef, LocalNotification, NotificationManagerRef,
 };
 use crate::model::FragmentId;
-use crate::storage::MetaStore;
 
 pub type ServingVnodeMappingRef = Arc<ServingVnodeMapping>;
 
@@ -104,10 +103,10 @@ fn to_deleted_fragment_parallel_unit_mapping(
         .collect()
 }
 
-pub(crate) async fn on_meta_start<S: MetaStore>(
-    notification_manager: NotificationManagerRef<S>,
-    cluster_manager: ClusterManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
+pub(crate) async fn on_meta_start(
+    notification_manager: NotificationManagerRef,
+    cluster_manager: ClusterManagerRef,
+    fragment_manager: FragmentManagerRef,
     serving_vnode_mapping: ServingVnodeMappingRef,
 ) {
     let streaming_parallelisms = fragment_manager.running_fragment_parallelisms(None).await;
@@ -127,10 +126,10 @@ pub(crate) async fn on_meta_start<S: MetaStore>(
     );
 }
 
-pub(crate) async fn start_serving_vnode_mapping_worker<S: MetaStore>(
-    notification_manager: NotificationManagerRef<S>,
-    cluster_manager: ClusterManagerRef<S>,
-    fragment_manager: FragmentManagerRef<S>,
+pub(crate) async fn start_serving_vnode_mapping_worker(
+    notification_manager: NotificationManagerRef,
+    cluster_manager: ClusterManagerRef,
+    fragment_manager: FragmentManagerRef,
     serving_vnode_mapping: ServingVnodeMappingRef,
 ) -> (JoinHandle<()>, Sender<()>) {
     let (local_notification_tx, mut local_notification_rx) = tokio::sync::mpsc::unbounded_channel();
diff --git a/src/meta/src/storage/meta_store.rs b/src/meta/src/storage/meta_store.rs
index 8a9e4d4e88600..123225bb7a3dd 100644
--- a/src/meta/src/storage/meta_store.rs
+++ b/src/meta/src/storage/meta_store.rs
@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::ops::Deref;
+use std::sync::Arc;
+
 use async_trait::async_trait;
 use risingwave_common::config::MetaBackend;
 use thiserror::Error;
@@ -27,9 +30,22 @@ pub trait Snapshot: Sync + Send + 'static {
     async fn get_cf(&self, cf: &str, key: &[u8]) -> MetaStoreResult<Vec<u8>>;
 }
 
+pub struct BoxedSnapshot(Box<dyn Snapshot + Send + Sync>);
+
+#[async_trait]
+impl Snapshot for BoxedSnapshot {
+    async fn list_cf(&self, cf: &str) -> MetaStoreResult<Vec<(Vec<u8>, Vec<u8>)>> {
+        self.0.deref().list_cf(cf).await
+    }
+
+    async fn get_cf(&self, cf: &str, key: &[u8]) -> MetaStoreResult<Vec<u8>> {
+        self.0.deref().get_cf(cf, key).await
+    }
+}
+
 /// `MetaStore` defines the functions used to operate metadata.
 #[async_trait]
-pub trait MetaStore: Clone + Sync + Send + 'static {
+pub trait MetaStore: Sync + Send + 'static {
     type Snapshot: Snapshot;
 
     async fn snapshot(&self) -> Self::Snapshot;
@@ -49,6 +65,90 @@ pub trait MetaStore: Clone + Sync + Send + 'static {
     fn meta_store_type(&self) -> MetaBackend;
 }
 
+#[derive(Clone)]
+pub struct MetaStoreRef(Arc<dyn MetaStore<Snapshot = BoxedSnapshot> + Send + Sync>);
+
+pub struct BoxedSnapshotMetaStore<S: MetaStore> {
+    inner: S,
+}
+
+pub trait MetaStoreBoxExt: MetaStore
+where
+    Self: Sized,
+{
+    fn into_ref(self) -> MetaStoreRef {
+        MetaStoreRef(Arc::new(BoxedSnapshotMetaStore { inner: self }))
+    }
+}
+
+impl<S: MetaStore> MetaStoreBoxExt for S {}
+
+#[async_trait]
+impl<S: MetaStore> MetaStore for BoxedSnapshotMetaStore<S> {
+    type Snapshot = BoxedSnapshot;
+
+    async fn snapshot(&self) -> Self::Snapshot {
+        BoxedSnapshot(Box::new(self.inner.snapshot().await))
+    }
+
+    async fn put_cf(&self, cf: &str, key: Key, value: Value) -> MetaStoreResult<()> {
+        self.inner.put_cf(cf, key, value).await
+    }
+
+    async fn delete_cf(&self, cf: &str, key: &[u8]) -> MetaStoreResult<()> {
+        self.inner.delete_cf(cf, key).await
+    }
+
+    async fn txn(&self, trx: Transaction) -> MetaStoreResult<()> {
+        self.inner.txn(trx).await
+    }
+
+    async fn list_cf(&self, cf: &str) -> MetaStoreResult<Vec<(Vec<u8>, Vec<u8>)>> {
+        self.inner.list_cf(cf).await
+    }
+
+    async fn get_cf(&self, cf: &str, key: &[u8]) -> MetaStoreResult<Vec<u8>> {
+        self.inner.get_cf(cf, key).await
+    }
+
+    fn meta_store_type(&self) -> MetaBackend {
+        self.inner.meta_store_type()
+    }
+}
+
+#[async_trait]
+impl MetaStore for MetaStoreRef {
+    type Snapshot = BoxedSnapshot;
+
+    async fn snapshot(&self) -> BoxedSnapshot {
+        self.0.deref().snapshot().await
+    }
+
+    async fn put_cf(&self, cf: &str, key: Key, value: Value) -> MetaStoreResult<()> {
+        self.0.deref().put_cf(cf, key, value).await
+    }
+
+    async fn delete_cf(&self, cf: &str, key: &[u8]) -> MetaStoreResult<()> {
+        self.0.deref().delete_cf(cf, key).await
+    }
+
+    async fn txn(&self, trx: Transaction) -> MetaStoreResult<()> {
+        self.0.deref().txn(trx).await
+    }
+
+    async fn list_cf(&self, cf: &str) -> MetaStoreResult<Vec<(Vec<u8>, Vec<u8>)>> {
+        self.0.deref().list_cf(cf).await
+    }
+
+    async fn get_cf(&self, cf: &str, key: &[u8]) -> MetaStoreResult<Vec<u8>> {
+        self.0.deref().get_cf(cf, key).await
+    }
+
+    fn meta_store_type(&self) -> MetaBackend {
+        self.0.deref().meta_store_type()
+    }
+}
+
 // Error of metastore
 #[derive(Debug, Error)]
 pub enum MetaStoreError {
diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs
index 39b113a14ac48..97efd9f05531d 100644
--- a/src/meta/src/stream/scale.rs
+++ b/src/meta/src/stream/scale.rs
@@ -16,7 +16,7 @@ use std::cmp::{min, Ordering};
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
 use std::iter::repeat;
 
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use futures::future::BoxFuture;
 use itertools::Itertools;
 use num_integer::Integer;
@@ -40,7 +40,7 @@ use uuid::Uuid;
 use crate::barrier::{Command, Reschedule};
 use crate::manager::{IdCategory, WorkerId};
 use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments};
-use crate::storage::{MetaStore, MetaStoreError, Transaction, DEFAULT_COLUMN_FAMILY};
+use crate::storage::{MetaStore, MetaStoreError, MetaStoreRef, Transaction, DEFAULT_COLUMN_FAMILY};
 use crate::stream::GlobalStreamManager;
 use crate::{MetaError, MetaResult};
 
@@ -56,10 +56,7 @@ impl From<TableRevision> for u64 {
 }
 
 impl TableRevision {
-    pub async fn get<S>(store: &S) -> MetaResult<Self>
-    where
-        S: MetaStore,
-    {
+    pub async fn get(store: &MetaStoreRef) -> MetaResult<Self> {
         let version = match store
             .get_cf(DEFAULT_COLUMN_FAMILY, TABLE_REVISION_KEY)
             .await
@@ -349,10 +346,7 @@ pub(crate) fn rebalance_actor_vnode(
     result
 }
 
-impl<S> GlobalStreamManager<S>
-where
-    S: MetaStore,
-{
+impl GlobalStreamManager {
     /// Build the context for rescheduling and do some validation for the request.
     async fn build_reschedule_context(
         &self,
@@ -1016,7 +1010,10 @@ where
                     let upstream_worker_id = ctx
                         .actor_id_to_parallel_unit(upstream_actor_id)?
                         .worker_node_id;
-                    let upstream_worker = ctx.worker_nodes.get(&upstream_worker_id).unwrap();
+                    let upstream_worker =
+                        ctx.worker_nodes.get(&upstream_worker_id).with_context(|| {
+                            format!("upstream worker {} not found", upstream_worker_id)
+                        })?;
 
                     // Force broadcast upstream actor info, because the actor information of the new
                     // node may not have been synchronized yet
@@ -1040,8 +1037,12 @@ where
                         let downstream_worker_id = ctx
                             .actor_id_to_parallel_unit(downstream_actor_id)?
                             .worker_node_id;
-                        let downstream_worker =
-                            ctx.worker_nodes.get(&downstream_worker_id).unwrap();
+                        let downstream_worker = ctx
+                            .worker_nodes
+                            .get(&downstream_worker_id)
+                            .with_context(|| {
+                                format!("downstream worker {} not found", downstream_worker_id)
+                            })?;
 
                         actor_infos_to_broadcast.insert(
                             *downstream_actor_id,
@@ -1271,7 +1272,8 @@ where
                 let worker_id = ctx
                     .parallel_unit_id_to_worker_id
                     .get(parallel_unit_id)
-                    .unwrap();
+                    .with_context(|| format!("parallel unit {} not found", parallel_unit_id))?;
+
                 created_actors.insert(
                     *actor_id,
                     (
@@ -1308,7 +1310,7 @@ where
         tracing::debug!("reschedule plan: {:#?}", reschedule_fragment);
 
         self.barrier_scheduler
-            .run_command_with_paused(Command::RescheduleFragment {
+            .run_config_change_command_with_pause(Command::RescheduleFragment {
                 reschedules: reschedule_fragment,
             })
             .await?;
@@ -1587,10 +1589,7 @@ where
     }
 }
 
-impl<S> GlobalStreamManager<S>
-where
-    S: MetaStore,
-{
+impl GlobalStreamManager {
     async fn generate_stable_resize_plan(
         &self,
         policy: StableResizePolicy,
@@ -1681,6 +1680,7 @@ where
             include_worker_ids: BTreeSet<WorkerId>,
             exclude_worker_ids: BTreeSet<WorkerId>,
             target_parallelism: Option<usize>,
+            target_parallelism_per_worker: Option<usize>,
         }
 
         let mut fragment_worker_changes: HashMap<_, _> = fragment_worker_changes
@@ -1692,6 +1692,9 @@ where
                         include_worker_ids: changes.include_worker_ids.into_iter().collect(),
                         exclude_worker_ids: changes.exclude_worker_ids.into_iter().collect(),
                         target_parallelism: changes.target_parallelism.map(|p| p as usize),
+                        target_parallelism_per_worker: changes
+                            .target_parallelism_per_worker
+                            .map(|p| p as usize),
                     },
                 )
             })
@@ -1710,6 +1713,7 @@ where
                 include_worker_ids,
                 exclude_worker_ids,
                 target_parallelism,
+                target_parallelism_per_worker,
             },
         ) in fragment_worker_changes
         {
@@ -1749,19 +1753,69 @@ where
                 })
                 .collect();
 
+            let include_worker_parallel_unit_ids = include_worker_ids
+                .iter()
+                .flat_map(|worker_id| worker_parallel_units.get(worker_id).unwrap())
+                .cloned()
+                .collect_vec();
+
+            let exclude_worker_parallel_unit_ids = exclude_worker_ids
+                .iter()
+                .flat_map(|worker_id| worker_parallel_units.get(worker_id).unwrap())
+                .cloned()
+                .collect_vec();
+
+            fn refilter_parallel_unit_id_by_target_parallelism(
+                worker_parallel_units: &HashMap<u32, HashSet<ParallelUnitId>>,
+                include_worker_ids: &BTreeSet<WorkerId>,
+                include_worker_parallel_unit_ids: &[ParallelUnitId],
+                target_parallel_unit_ids: &mut BTreeSet<ParallelUnitId>,
+                target_parallelism_per_worker: usize,
+            ) {
+                let limited_worker_parallel_unit_ids = include_worker_ids
+                    .iter()
+                    .flat_map(|worker_id| {
+                        worker_parallel_units
+                            .get(worker_id)
+                            .cloned()
+                            .unwrap()
+                            .into_iter()
+                            .sorted()
+                            .take(target_parallelism_per_worker)
+                    })
+                    .collect_vec();
+
+                // remove all the parallel units in the limited workers
+                target_parallel_unit_ids
+                    .retain(|id| !include_worker_parallel_unit_ids.contains(id));
+
+                // then we re-add the limited parallel units from the limited workers
+                target_parallel_unit_ids.extend(limited_worker_parallel_unit_ids.into_iter());
+            }
+
             match fragment.get_distribution_type().unwrap() {
                 FragmentDistributionType::Unspecified => unreachable!(),
                 FragmentDistributionType::Single => {
                     let single_parallel_unit_id =
                         fragment_parallel_unit_ids.iter().exactly_one().unwrap();
 
-                    let target_parallel_unit_ids: BTreeSet<_> = worker_parallel_units
+                    let mut target_parallel_unit_ids: BTreeSet<_> = worker_parallel_units
                         .keys()
                         .filter(|id| !unschedulable_worker_ids.contains(*id))
                         .filter(|id| !exclude_worker_ids.contains(*id))
                         .flat_map(|id| worker_parallel_units.get(id).cloned().unwrap())
                         .collect();
 
+                    if let Some(target_parallelism_per_worker) = target_parallelism_per_worker {
+                        refilter_parallel_unit_id_by_target_parallelism(
+                            &worker_parallel_units,
+                            &include_worker_ids,
+                            &include_worker_parallel_unit_ids,
+                            &mut target_parallel_unit_ids,
+                            target_parallelism_per_worker,
+                        );
+                    }
+
                     if target_parallel_unit_ids.is_empty() {
                         bail!(
                             "No schedulable ParallelUnits available for single distribution fragment {}",
@@ -1788,18 +1842,6 @@ where
                     }
                 }
                 FragmentDistributionType::Hash => {
-                    let include_worker_parallel_unit_ids = include_worker_ids
-                        .iter()
-                        .flat_map(|worker_id| worker_parallel_units.get(worker_id).unwrap())
-                        .cloned()
-                        .collect_vec();
-
-                    let exclude_worker_parallel_unit_ids = exclude_worker_ids
-                        .iter()
-                        .flat_map(|worker_id| worker_parallel_units.get(worker_id).unwrap())
-                        .cloned()
-                        .collect_vec();
-
                     let mut target_parallel_unit_ids: BTreeSet<_> =
                         fragment_parallel_unit_ids.clone();
                     target_parallel_unit_ids.extend(include_worker_parallel_unit_ids.iter());
@@ -1813,15 +1855,30 @@ where
                         );
                     }
 
-                    if let Some(target_parallelism) = target_parallelism {
-                        if target_parallel_unit_ids.len() < target_parallelism {
-                            bail!("Target parallelism {} is greater than schedulable ParallelUnits {}", target_parallelism, target_parallel_unit_ids.len());
+                    match (target_parallelism, target_parallelism_per_worker) {
+                        (Some(_), Some(_)) => {
+                            bail!("Cannot specify both target parallelism and target parallelism per worker");
                         }
+                        (Some(target_parallelism), _) => {
+                            if target_parallel_unit_ids.len() < target_parallelism {
+                                bail!("Target parallelism {} is greater than schedulable ParallelUnits {}", target_parallelism, target_parallel_unit_ids.len());
+                            }
 
-                        target_parallel_unit_ids = target_parallel_unit_ids
-                            .into_iter()
-                            .take(target_parallelism)
-                            .collect();
+                            target_parallel_unit_ids = target_parallel_unit_ids
+                                .into_iter()
+                                .take(target_parallelism)
+                                .collect();
+                        }
+                        (_, Some(target_parallelism_per_worker)) => {
+                            refilter_parallel_unit_id_by_target_parallelism(
+                                &worker_parallel_units,
+                                &include_worker_ids,
+                                &include_worker_parallel_unit_ids,
+                                &mut target_parallel_unit_ids,
+                                target_parallelism_per_worker,
+                            );
+                        }
+                        _ => {}
                     }
 
                     let to_expand_parallel_units = target_parallel_unit_ids
diff --git a/src/meta/src/stream/source_manager.rs b/src/meta/src/stream/source_manager.rs
index 43f56bae8703b..a6b25d5fba4d7 100644
--- a/src/meta/src/stream/source_manager.rs
+++ b/src/meta/src/stream/source_manager.rs
@@ -40,17 +40,16 @@ use crate::barrier::{BarrierScheduler, Command};
 use crate::manager::{CatalogManagerRef, FragmentManagerRef, MetaSrvEnv, SourceId};
 use crate::model::{ActorId, FragmentId, TableFragments};
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
-pub type SourceManagerRef<S> = Arc<SourceManager<S>>;
+pub type SourceManagerRef = Arc<SourceManager>;
 pub type SplitAssignment = HashMap<FragmentId, HashMap<ActorId, Vec<SplitImpl>>>;
 
-pub struct SourceManager<S: MetaStore> {
+pub struct SourceManager {
     pub(crate) paused: Mutex<()>,
-    env: MetaSrvEnv<S>,
-    barrier_scheduler: BarrierScheduler<S>,
-    core: Mutex<SourceManagerCore<S>>,
+    env: MetaSrvEnv,
+    barrier_scheduler: BarrierScheduler,
+    core: Mutex<SourceManagerCore>,
     metrics: Arc<MetaMetrics>,
 }
 
@@ -105,7 +104,7 @@ impl ConnectorSourceWorker {
         let mut properties = ConnectorProperties::extract(source.properties.clone())?;
         if properties.is_cdc_connector() {
             let table_schema = Self::extract_source_schema(source);
-            properties.init_cdc_properties(Some(table_schema));
+            properties.init_cdc_properties(table_schema);
         }
         let enumerator = SplitEnumeratorImpl::create(
             properties.clone(),
@@ -222,8 +221,8 @@ impl ConnectorSourceWorkerHandle {
     }
 }
 
-pub struct SourceManagerCore<S: MetaStore> {
-    fragment_manager: FragmentManagerRef<S>,
+pub struct SourceManagerCore {
+    fragment_manager: FragmentManagerRef,
 
     /// Managed source loops
     managed_sources: HashMap<SourceId, ConnectorSourceWorkerHandle>,
@@ -236,12 +235,9 @@ pub struct SourceManagerCore<S: MetaStore> {
     actor_splits: HashMap<ActorId, Vec<SplitImpl>>,
 }
 
-impl<S> SourceManagerCore<S>
-where
-    S: MetaStore,
-{
+impl SourceManagerCore {
     fn new(
-        fragment_manager: FragmentManagerRef<S>,
+        fragment_manager: FragmentManagerRef,
         managed_sources: HashMap<SourceId, ConnectorSourceWorkerHandle>,
         source_fragments: HashMap<SourceId, BTreeSet<FragmentId>>,
         actor_splits: HashMap<ActorId, Vec<SplitImpl>>,
@@ -510,18 +506,15 @@ where
     )
 }
 
-impl<S> SourceManager<S>
-where
-    S: MetaStore,
-{
+impl SourceManager {
     const DEFAULT_SOURCE_TICK_INTERVAL: Duration = Duration::from_secs(10);
     const DEFAULT_SOURCE_TICK_TIMEOUT: Duration = Duration::from_secs(10);
 
     pub async fn new(
-        env: MetaSrvEnv<S>,
-        barrier_scheduler: BarrierScheduler<S>,
-        catalog_manager: CatalogManagerRef<S>,
-        fragment_manager: FragmentManagerRef<S>,
+        env: MetaSrvEnv,
+        barrier_scheduler: BarrierScheduler,
+        catalog_manager: CatalogManagerRef,
+        fragment_manager: FragmentManagerRef,
         metrics: Arc<MetaMetrics>,
     ) -> MetaResult<Self> {
         let mut managed_sources = HashMap::new();
diff --git a/src/meta/src/stream/stream_graph/actor.rs b/src/meta/src/stream/stream_graph/actor.rs
index 894dbb2223c4d..8851e2e880da6 100644
--- a/src/meta/src/stream/stream_graph/actor.rs
+++ b/src/meta/src/stream/stream_graph/actor.rs
@@ -33,7 +33,6 @@ use super::id::GlobalFragmentIdsExt;
 use super::Locations;
 use crate::manager::{IdGeneratorManagerRef, StreamingClusterInfo, StreamingJob};
 use crate::model::{DispatcherId, FragmentId};
-use crate::storage::MetaStore;
 use crate::stream::stream_graph::fragment::{
     CompleteStreamFragmentGraph, EdgeId, EitherFragment, StreamFragmentEdge,
 };
@@ -656,14 +655,11 @@ impl ActorGraphBuilder {
 
     /// Build a stream graph by duplicating each fragment as parallel actors. Returns
     /// [`ActorGraphBuildResult`] that will be further used to build actors on the compute nodes.
-    pub async fn generate_graph<S>(
+    pub async fn generate_graph(
         self,
-        id_gen_manager: IdGeneratorManagerRef<S>,
+        id_gen_manager: IdGeneratorManagerRef,
         job: &StreamingJob,
-    ) -> MetaResult<ActorGraphBuildResult>
-    where
-        S: MetaStore,
-    {
+    ) -> MetaResult<ActorGraphBuildResult> {
         // Pre-generate IDs for all actors.
         let actor_len = self
             .distributions
diff --git a/src/meta/src/stream/stream_graph/fragment.rs b/src/meta/src/stream/stream_graph/fragment.rs
index 753b37ab846ca..5f022330f261c 100644
--- a/src/meta/src/stream/stream_graph/fragment.rs
+++ b/src/meta/src/stream/stream_graph/fragment.rs
@@ -37,7 +37,6 @@ use risingwave_pb::stream_plan::{
 
 use crate::manager::{IdGeneratorManagerRef, StreamingJob};
 use crate::model::FragmentId;
-use crate::storage::MetaStore;
 use crate::stream::stream_graph::id::{GlobalFragmentId, GlobalFragmentIdGen, GlobalTableIdGen};
 use crate::stream::stream_graph::schedule::Distribution;
 use crate::MetaResult;
@@ -263,9 +262,9 @@ pub struct StreamFragmentGraph {
 impl StreamFragmentGraph {
     /// Create a new [`StreamFragmentGraph`] from the given [`StreamFragmentGraphProto`], with all
     /// global IDs correctly filled.
-    pub async fn new<S: MetaStore>(
+    pub async fn new(
         proto: StreamFragmentGraphProto,
-        id_gen: IdGeneratorManagerRef<S>,
+        id_gen: IdGeneratorManagerRef,
         job: &StreamingJob,
     ) -> MetaResult<Self> {
         let fragment_id_gen =
@@ -440,7 +439,7 @@ static EMPTY_HASHMAP: LazyLock<HashMap<GlobalFragmentId, StreamFragmentEdge>> =
     LazyLock::new(HashMap::new);
 
 /// A fragment that is either being built or already exists. Used for generalize the logic of
-/// [`ActorGraphBuilder`].
+/// [`crate::stream::ActorGraphBuilder`].
 #[derive(Debug, Clone, EnumAsInner)]
 pub(super) enum EitherFragment {
     /// An internal fragment that is being built for the current streaming job.
diff --git a/src/meta/src/stream/stream_graph/id.rs b/src/meta/src/stream/stream_graph/id.rs
index b468637142150..34fca06690c0d 100644
--- a/src/meta/src/stream/stream_graph/id.rs
+++ b/src/meta/src/stream/stream_graph/id.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use crate::manager::{IdCategory, IdCategoryType, IdGeneratorManager};
-use crate::storage::MetaStore;
 use crate::MetaResult;
 
 /// A wrapper to distinguish global ID generated by the [`IdGeneratorManager`] and the local ID from
@@ -48,7 +47,7 @@ pub(super) struct GlobalIdGen<const TYPE: IdCategoryType> {
 
 impl<const TYPE: IdCategoryType> GlobalIdGen<TYPE> {
     /// Pre-allocate a range of IDs with the given `len` and return the generator.
-    pub async fn new<S: MetaStore>(id_gen: &IdGeneratorManager<S>, len: u64) -> MetaResult<Self> {
+    pub async fn new(id_gen: &IdGeneratorManager, len: u64) -> MetaResult<Self> {
         let offset = id_gen.generate_interval::<TYPE>(len).await?;
         Ok(Self {
             offset: offset as u32,
diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs
index 182c03aa2f11a..558149787c85f 100644
--- a/src/meta/src/stream/stream_manager.rs
+++ b/src/meta/src/stream/stream_manager.rs
@@ -34,11 +34,10 @@ use crate::barrier::{BarrierScheduler, Command};
 use crate::hummock::HummockManagerRef;
 use crate::manager::{ClusterManagerRef, FragmentManagerRef, MetaSrvEnv};
 use crate::model::{ActorId, TableFragments};
-use crate::storage::MetaStore;
 use crate::stream::SourceManagerRef;
 use crate::{MetaError, MetaResult};
 
-pub type GlobalStreamManagerRef<S> = Arc<GlobalStreamManager<S>>;
+pub type GlobalStreamManagerRef = Arc<GlobalStreamManager>;
 
 /// [`CreateStreamingJobContext`] carries one-time infos for creating a streaming job.
 ///
@@ -159,40 +158,37 @@ pub struct ReplaceTableContext {
 }
 
 /// `GlobalStreamManager` manages all the streams in the system.
-pub struct GlobalStreamManager<S: MetaStore> {
-    pub(crate) env: MetaSrvEnv<S>,
+pub struct GlobalStreamManager {
+    pub(crate) env: MetaSrvEnv,
 
     /// Manages definition and status of fragments and actors
-    pub(super) fragment_manager: FragmentManagerRef<S>,
+    pub(super) fragment_manager: FragmentManagerRef,
 
     /// Broadcasts and collect barriers
-    pub(crate) barrier_scheduler: BarrierScheduler<S>,
+    pub(crate) barrier_scheduler: BarrierScheduler,
 
     /// Maintains information of the cluster
-    pub(crate) cluster_manager: ClusterManagerRef<S>,
+    pub(crate) cluster_manager: ClusterManagerRef,
 
     /// Maintains streaming sources from external system like kafka
-    pub(crate) source_manager: SourceManagerRef<S>,
+    pub(crate) source_manager: SourceManagerRef,
 
     /// Creating streaming job info.
     creating_job_info: CreatingStreamingJobInfoRef,
 
-    hummock_manager: HummockManagerRef<S>,
+    hummock_manager: HummockManagerRef,
 
     pub(crate) reschedule_lock: RwLock<()>,
 }
 
-impl<S> GlobalStreamManager<S>
-where
-    S: MetaStore,
-{
+impl GlobalStreamManager {
     pub fn new(
-        env: MetaSrvEnv<S>,
-        fragment_manager: FragmentManagerRef<S>,
-        barrier_scheduler: BarrierScheduler<S>,
-        cluster_manager: ClusterManagerRef<S>,
-        source_manager: SourceManagerRef<S>,
-        hummock_manager: HummockManagerRef<S>,
+        env: MetaSrvEnv,
+        fragment_manager: FragmentManagerRef,
+        barrier_scheduler: BarrierScheduler,
+        cluster_manager: ClusterManagerRef,
+        source_manager: SourceManagerRef,
+        hummock_manager: HummockManagerRef,
     ) -> MetaResult<Self> {
         Ok(Self {
             env,
@@ -489,7 +485,7 @@ where
 
         if let Err(err) = self
             .barrier_scheduler
-            .run_command_with_paused(Command::ReplaceTable {
+            .run_config_change_command_with_pause(Command::ReplaceTable {
                 old_table_fragments,
                 new_table_fragments: table_fragments,
                 merge_updates,
@@ -612,7 +608,6 @@ mod tests {
     use crate::model::{ActorId, FragmentId};
     use crate::rpc::ddl_controller::DropMode;
     use crate::rpc::metrics::MetaMetrics;
-    use crate::storage::MemStore;
     use crate::stream::SourceManager;
     use crate::MetaOpts;
 
@@ -713,9 +708,9 @@ mod tests {
     }
 
     struct MockServices {
-        global_stream_manager: GlobalStreamManagerRef<MemStore>,
-        catalog_manager: CatalogManagerRef<MemStore>,
-        fragment_manager: FragmentManagerRef<MemStore>,
+        global_stream_manager: GlobalStreamManagerRef,
+        catalog_manager: CatalogManagerRef,
+        fragment_manager: FragmentManagerRef,
         state: Arc<FakeFragmentState>,
         join_handle_shutdown_txs: Vec<(JoinHandle<()>, Sender<()>)>,
     }
diff --git a/src/meta/src/telemetry.rs b/src/meta/src/telemetry.rs
index 4d62594ff7641..774b3cdda8146 100644
--- a/src/meta/src/telemetry.rs
+++ b/src/meta/src/telemetry.rs
@@ -25,7 +25,6 @@ use serde::{Deserialize, Serialize};
 
 use crate::manager::ClusterManager;
 use crate::model::ClusterId;
-use crate::storage::MetaStore;
 
 #[derive(Debug, Serialize, Deserialize)]
 struct NodeCount {
@@ -64,13 +63,13 @@ impl TelemetryInfoFetcher for MetaTelemetryInfoFetcher {
 }
 
 #[derive(Clone)]
-pub(crate) struct MetaReportCreator<S: MetaStore> {
-    cluster_mgr: Arc<ClusterManager<S>>,
+pub(crate) struct MetaReportCreator {
+    cluster_mgr: Arc<ClusterManager>,
     meta_backend: MetaBackend,
 }
 
-impl<S: MetaStore> MetaReportCreator<S> {
-    pub(crate) fn new(cluster_mgr: Arc<ClusterManager<S>>, meta_backend: MetaBackend) -> Self {
+impl MetaReportCreator {
+    pub(crate) fn new(cluster_mgr: Arc<ClusterManager>, meta_backend: MetaBackend) -> Self {
         Self {
             cluster_mgr,
             meta_backend,
@@ -79,7 +78,7 @@ impl<S: MetaStore> MetaReportCreator<S> {
 }
 
 #[async_trait::async_trait]
-impl<S: MetaStore> TelemetryReportCreator for MetaReportCreator<S> {
+impl TelemetryReportCreator for MetaReportCreator {
     async fn create_report(
         &self,
         tracking_id: String,
diff --git a/src/object_store/src/object/mod.rs b/src/object_store/src/object/mod.rs
index 51f1acff0c874..deb29a07dc484 100644
--- a/src/object_store/src/object/mod.rs
+++ b/src/object_store/src/object/mod.rs
@@ -17,7 +17,6 @@ use std::time::Duration;
 
 use bytes::Bytes;
 use prometheus::HistogramTimer;
-use risingwave_common::config::StorageConfig;
 use tokio::io::{AsyncRead, AsyncReadExt};
 
 pub mod mem;
@@ -848,11 +847,10 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> {
     }
 }
 
-pub async fn parse_remote_object_store_with_config(
+pub async fn parse_remote_object_store(
     url: &str,
     metrics: Arc<ObjectStoreMetrics>,
     ident: &str,
-    config: Option<Arc<StorageConfig>>,
 ) -> ObjectStoreImpl {
     match url {
         s3 if s3.starts_with("s3://") => ObjectStoreImpl::S3(
@@ -866,7 +864,7 @@ pub async fn parse_remote_object_store_with_config(
         #[cfg(feature = "hdfs-backend")]
         hdfs if hdfs.starts_with("hdfs://") => {
             let hdfs = hdfs.strip_prefix("hdfs://").unwrap();
-            let (namenode, root) = hdfs.split_once('@').unwrap();
+            let (namenode, root) = hdfs.split_once('@').unwrap_or((hdfs, ""));
             ObjectStoreImpl::Opendal(
                 OpendalObjectStore::new_hdfs_engine(namenode.to_string(), root.to_string())
                     .unwrap()
@@ -875,7 +873,7 @@ pub async fn parse_remote_object_store_with_config(
         }
         gcs if gcs.starts_with("gcs://") => {
             let gcs = gcs.strip_prefix("gcs://").unwrap();
-            let (bucket, root) = gcs.split_once('@').unwrap();
+            let (bucket, root) = gcs.split_once('@').unwrap_or((gcs, ""));
             ObjectStoreImpl::Opendal(
                 OpendalObjectStore::new_gcs_engine(bucket.to_string(), root.to_string())
                     .unwrap()
@@ -885,7 +883,7 @@ pub async fn parse_remote_object_store_with_config(
 
         oss if oss.starts_with("oss://") => {
             let oss = oss.strip_prefix("oss://").unwrap();
-            let (bucket, root) = oss.split_once('@').unwrap();
+            let (bucket, root) = oss.split_once('@').unwrap_or((oss, ""));
             ObjectStoreImpl::Opendal(
                 OpendalObjectStore::new_oss_engine(bucket.to_string(), root.to_string())
                     .unwrap()
@@ -894,63 +892,34 @@ pub async fn parse_remote_object_store_with_config(
         }
         webhdfs if webhdfs.starts_with("webhdfs://") => {
             let webhdfs = webhdfs.strip_prefix("webhdfs://").unwrap();
-            let (endpoint, root) = webhdfs.split_once('@').unwrap();
+            let (namenode, root) = webhdfs.split_once('@').unwrap_or((webhdfs, ""));
             ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_webhdfs_engine(endpoint.to_string(), root.to_string())
+                OpendalObjectStore::new_webhdfs_engine(namenode.to_string(), root.to_string())
                     .unwrap()
                     .monitored(metrics),
             )
         }
         azblob if azblob.starts_with("azblob://") => {
             let azblob = azblob.strip_prefix("azblob://").unwrap();
-            let (container_name, root) = azblob.split_once('@').unwrap();
+            let (container_name, root) = azblob.split_once('@').unwrap_or((azblob, ""));
             ObjectStoreImpl::Opendal(
                 OpendalObjectStore::new_azblob_engine(container_name.to_string(), root.to_string())
                     .unwrap()
                     .monitored(metrics),
             )
         }
-        fs if fs.starts_with("fs://") => {
-            let fs = fs.strip_prefix("fs://").unwrap();
-            let (_, root) = fs.split_once('@').unwrap();
-            ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_fs_engine(root.to_string())
-                    .unwrap()
-                    .monitored(metrics),
-            )
-        }
+        fs if fs.starts_with("fs://") => ObjectStoreImpl::Opendal(
+            // Now fs engine is only used in CI, so we can hardcode root.
+            OpendalObjectStore::new_fs_engine("/tmp/rw_ci".to_string())
+                .unwrap()
+                .monitored(metrics),
+        ),
 
         s3_compatible if s3_compatible.starts_with("s3-compatible://") => {
-            let s3_object_store_config = config
-                .map(|storage_config| S3ObjectStoreConfig {
-                    keepalive_ms: storage_config.object_store_keepalive_ms,
-                    recv_buffer_size: storage_config.object_store_recv_buffer_size,
-                    send_buffer_size: storage_config.object_store_send_buffer_size,
-                    nodelay: storage_config.object_store_nodelay,
-                    req_retry_interval_ms: Some(storage_config.object_store_req_retry_interval_ms),
-                    req_retry_max_delay_ms: Some(
-                        storage_config.object_store_req_retry_max_delay_ms,
-                    ),
-                    req_retry_max_attempts: Some(
-                        storage_config.object_store_req_retry_max_attempts,
-                    ),
-                })
-                .unwrap_or_default();
-
-            ObjectStoreImpl::S3(
-                // For backward compatibility, s3-compatible is still reserved.
-                // todo: remove this after this change has been applied for downstream projects.
-                S3ObjectStore::new_with_config(
-                    s3_compatible
-                        .strip_prefix("s3-compatible://")
-                        .unwrap()
-                        .to_string(),
-                    metrics.clone(),
-                    s3_object_store_config,
-                )
-                .await
-                .monitored(metrics),
-            )
+            tracing::error!("The s3 compatible mode has been unified with s3.");
+            tracing::error!("If you want to use s3 compatible storage, please set your access_key, secret_key and region to the environment variable AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION,
+            set your endpoint to the environment variable RW_S3_ENDPOINT.");
+            panic!("Passing s3-compatible is not supported, please modify the environment variable and pass in s3.");
         }
         minio if minio.starts_with("minio://") => ObjectStoreImpl::S3(
             S3ObjectStore::with_minio(minio, metrics.clone())
@@ -982,12 +951,4 @@ pub async fn parse_remote_object_store_with_config(
     }
 }
 
-pub async fn parse_remote_object_store(
-    url: &str,
-    metrics: Arc<ObjectStoreMetrics>,
-    ident: &str,
-) -> ObjectStoreImpl {
-    parse_remote_object_store_with_config(url, metrics, ident, None).await
-}
-
 pub type ObjectMetadataIter = BoxStream<'static, ObjectResult<ObjectMetadata>>;
diff --git a/src/object_store/src/object/opendal_engine/gcs.rs b/src/object_store/src/object/opendal_engine/gcs.rs
index 206a5ab6b4e08..bb6ef8eee0446 100644
--- a/src/object_store/src/object/opendal_engine/gcs.rs
+++ b/src/object_store/src/object/opendal_engine/gcs.rs
@@ -18,6 +18,13 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+/// The fixed number of bytes that is buffered before they are uploaded as a part, will be used in
+/// streaing upload.
+///
+/// Reference: <https://cloud.google.com/storage/docs/streaming-uploads>
+const GCS_PART_SIZE: usize = 16 * 1024 * 1024;
+
 impl OpendalObjectStore {
     /// create opendal gcs engine.
     pub fn new_gcs_engine(bucket: String, root: String) -> ObjectResult<Self> {
@@ -28,6 +35,8 @@ impl OpendalObjectStore {
 
         builder.root(&root);
 
+        builder.write_fixed_size(GCS_PART_SIZE);
+
         // if credential env is set, use it. Otherwise, ADC will be used.
         let cred = std::env::var("GOOGLE_APPLICATION_CREDENTIALS");
         if let Ok(cred) = cred {
diff --git a/src/object_store/src/object/opendal_engine/hdfs.rs b/src/object_store/src/object/opendal_engine/hdfs.rs
index 569c9fb91b62f..b9793968df5bb 100644
--- a/src/object_store/src/object/opendal_engine/hdfs.rs
+++ b/src/object_store/src/object/opendal_engine/hdfs.rs
@@ -25,8 +25,6 @@ impl OpendalObjectStore {
         let mut builder = Hdfs::default();
         // Set the name node for hdfs.
         builder.name_node(&namenode);
-        // Set the root for hdfs, all operations will happen under this root.
-        // NOTE: the root must be absolute path.
         builder.root(&root);
 
         let op: Operator = Operator::new(builder)?
diff --git a/src/object_store/src/object/opendal_engine/opendal_object_store.rs b/src/object_store/src/object/opendal_engine/opendal_object_store.rs
index 370cb3da5e011..de410507da622 100644
--- a/src/object_store/src/object/opendal_engine/opendal_object_store.rs
+++ b/src/object_store/src/object/opendal_engine/opendal_object_store.rs
@@ -193,7 +193,10 @@ impl StreamingUploader for OpenDalStreamingUploader {
     async fn finish(mut self: Box<Self>) -> ObjectResult<()> {
         match self.writer.close().await {
             Ok(_) => (),
-            Err(_) => self.writer.abort().await?,
+            Err(err) => {
+                self.writer.abort().await?;
+                return Err(err.into());
+            }
         };
 
         Ok(())
diff --git a/src/object_store/src/object/opendal_engine/oss.rs b/src/object_store/src/object/opendal_engine/oss.rs
index 292ddf2c614f1..f10458402ec76 100644
--- a/src/object_store/src/object/opendal_engine/oss.rs
+++ b/src/object_store/src/object/opendal_engine/oss.rs
@@ -18,6 +18,13 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+/// The minimum number of bytes that is buffered before they are uploaded as a part, , will be used
+/// in streaing upload.
+///
+/// Reference: <https://www.alibabacloud.com/help/en/oss/user-guide/multipart-upload-12>
+const OSS_PART_SIZE: usize = 16 * 1024 * 1024;
+
 impl OpendalObjectStore {
     /// create opendal oss engine.
     pub fn new_oss_engine(bucket: String, root: String) -> ObjectResult<Self> {
@@ -26,6 +33,8 @@ impl OpendalObjectStore {
 
         builder.bucket(&bucket);
 
+        builder.write_min_size(OSS_PART_SIZE);
+
         builder.root(&root);
 
         let endpoint = std::env::var("OSS_ENDPOINT")
diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs
index 72af0cb069359..e0d561e3bdb32 100644
--- a/src/object_store/src/object/s3.rs
+++ b/src/object_store/src/object/s3.rs
@@ -30,8 +30,7 @@ use aws_sdk_s3::types::{
     CompletedPart, Delete, ExpirationStatus, LifecycleRule, LifecycleRuleFilter, ObjectIdentifier,
 };
 use aws_sdk_s3::Client;
-use aws_smithy_client::conns::NativeTls;
-use aws_smithy_client::http_connector::ConnectorSettings;
+use aws_smithy_client::http_connector::{ConnectorSettings, HttpConnector};
 use aws_smithy_http::body::SdkBody;
 use aws_smithy_http::result::SdkError;
 use aws_smithy_types::retry::RetryConfig;
@@ -41,6 +40,7 @@ use futures::{stream, Stream};
 use hyper::Body;
 use itertools::Itertools;
 use risingwave_common::config::default::s3_objstore_config;
+use risingwave_common::monitor::connection::monitor_connector;
 use tokio::io::AsyncRead;
 use tokio::task::JoinHandle;
 use tokio_retry::strategy::{jitter, ExponentialBackoff};
@@ -543,13 +543,9 @@ impl S3ObjectStore {
         Self::new_with_config(bucket, metrics, S3ObjectStoreConfig::default()).await
     }
 
-    pub async fn new_with_config(
-        bucket: String,
-        metrics: Arc<ObjectStoreMetrics>,
-        config: S3ObjectStoreConfig,
-    ) -> Self {
+    pub fn new_http_connector(config: &S3ObjectStoreConfig) -> impl Into<HttpConnector> {
         // Customize http connector to set keepalive.
-        let native_tls = || -> NativeTls {
+        let native_tls = {
             let mut tls = hyper_tls::native_tls::TlsConnector::builder();
             let tls = tls
                 .min_protocol_version(Some(hyper_tls::native_tls::Protocol::Tlsv12))
@@ -577,13 +573,21 @@ impl S3ObjectStore {
             http.enforce_http(false);
             hyper_tls::HttpsConnector::from((http, tls.into()))
         };
-        let hyper_adapter = aws_smithy_client::hyper_ext::Adapter::builder()
+
+        aws_smithy_client::hyper_ext::Adapter::builder()
             .hyper_builder(hyper::client::Builder::default())
             .connector_settings(ConnectorSettings::builder().build())
-            .build(native_tls());
+            .build(monitor_connector(native_tls, "S3"))
+    }
+
+    pub async fn new_with_config(
+        bucket: String,
+        metrics: Arc<ObjectStoreMetrics>,
+        config: S3ObjectStoreConfig,
+    ) -> Self {
         let sdk_config_loader = aws_config::from_env()
             .retry_config(RetryConfig::standard().with_max_attempts(4))
-            .http_connector(hyper_adapter);
+            .http_connector(Self::new_http_connector(&config));
 
         // Retry 3 times if we get server-side errors or throttling errors
         let client = match std::env::var("RW_S3_ENDPOINT") {
@@ -635,7 +639,8 @@ impl S3ObjectStore {
         #[cfg(not(madsim))]
         let builder =
             aws_sdk_s3::config::Builder::from(&aws_config::ConfigLoader::default().load().await)
-                .force_path_style(true);
+                .force_path_style(true)
+                .http_connector(Self::new_http_connector(&S3ObjectStoreConfig::default()));
 
         let config = builder
             .region(Region::new("custom"))
diff --git a/src/prost/build.rs b/src/prost/build.rs
index 78097021fbd86..172f9c0731a6d 100644
--- a/src/prost/build.rs
+++ b/src/prost/build.rs
@@ -74,6 +74,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .type_attribute(".", "#[derive(prost_helpers::AnyPB)]")
         .type_attribute("node_body", "#[derive(::enum_as_inner::EnumAsInner)]")
         .type_attribute("rex_node", "#[derive(::enum_as_inner::EnumAsInner)]")
+        .type_attribute(
+            "meta.PausedReason",
+            "#[derive(::enum_as_inner::EnumAsInner)]",
+        )
         .type_attribute(
             "stream_plan.Barrier.BarrierKind",
             "#[derive(::enum_as_inner::EnumAsInner)]",
@@ -97,6 +101,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .type_attribute("plan_common.GeneratedColumnDesc", "#[derive(Eq, Hash)]")
         .type_attribute("plan_common.DefaultColumnDesc", "#[derive(Eq, Hash)]")
         .type_attribute("plan_common.Cardinality", "#[derive(Eq, Hash, Copy)]")
+        .type_attribute("plan_common.StorageTableDesc", "#[derive(Eq, Hash)]")
+        .type_attribute("plan_common.ColumnDesc", "#[derive(Eq, Hash)]")
+        .type_attribute("common.ColumnOrder", "#[derive(Eq, Hash)]")
+        .type_attribute("common.OrderType", "#[derive(Eq, Hash)]")
         // ===================
         .out_dir(out_dir.as_path())
         .compile(&protos, &[proto_dir.to_string()])
diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs
index c534a10b18ecb..a5e0cf0e82664 100644
--- a/src/prost/src/lib.rs
+++ b/src/prost/src/lib.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 #![expect(clippy::all)]
-#![expect(rustdoc::bare_urls)]
 #![expect(clippy::doc_markdown)]
 #![allow(non_snake_case)] // for derived code of `Message`
 #![feature(lint_reasons)]
diff --git a/src/risedevtool/Cargo.toml b/src/risedevtool/Cargo.toml
index 08a2bef7419c7..b50ca017c1149 100644
--- a/src/risedevtool/Cargo.toml
+++ b/src/risedevtool/Cargo.toml
@@ -20,7 +20,7 @@ clap = { version = "4", features = ["derive"] }
 console = "0.15"
 fs-err = "2.9.0"
 glob = "0.3"
-google-cloud-pubsub = "0.19"
+google-cloud-pubsub = "0.20"
 indicatif = "0.17"
 itertools = "0.11"
 rdkafka = { workspace = true }
diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs
index acb13473289ee..c2e586802489b 100644
--- a/src/risedevtool/src/bin/risedev-dev.rs
+++ b/src/risedevtool/src/bin/risedev-dev.rs
@@ -458,7 +458,7 @@ fn main() -> Result<()> {
                 err.root_cause().to_string().trim(),
             );
             println!(
-                "* Use `{}` to enable new compoenents, if they are missing.",
+                "* Use `{}` to enable new components, if they are missing.",
                 style("./risedev configure").blue().bold(),
             );
             println!(
diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs
index 05861a8b72caa..8890f984971fe 100644
--- a/src/risedevtool/src/service_config.rs
+++ b/src/risedevtool/src/service_config.rs
@@ -243,7 +243,6 @@ pub struct OpendalConfig {
     pub engine: String,
     pub namenode: String,
     pub bucket: String,
-    pub root: String,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
diff --git a/src/risedevtool/src/task/compactor_service.rs b/src/risedevtool/src/task/compactor_service.rs
index 904dbf964eb47..d94083745154e 100644
--- a/src/risedevtool/src/task/compactor_service.rs
+++ b/src/risedevtool/src/task/compactor_service.rs
@@ -55,7 +55,7 @@ impl CompactorService {
             .arg("--advertise-addr")
             .arg(format!("{}:{}", config.address, config.port))
             .arg("--metrics-level")
-            .arg("1");
+            .arg("info");
         if let Some(compaction_worker_threads_number) =
             config.compaction_worker_threads_number.as_ref()
         {
diff --git a/src/risedevtool/src/task/compute_node_service.rs b/src/risedevtool/src/task/compute_node_service.rs
index e9330739282f1..6c705154e0578 100644
--- a/src/risedevtool/src/task/compute_node_service.rs
+++ b/src/risedevtool/src/task/compute_node_service.rs
@@ -57,7 +57,7 @@ impl ComputeNodeService {
             .arg("--advertise-addr")
             .arg(format!("{}:{}", config.address, config.port))
             .arg("--metrics-level")
-            .arg("1")
+            .arg("info")
             .arg("--async-stack-trace")
             .arg(&config.async_stack_trace)
             .arg("--connector-rpc-endpoint")
diff --git a/src/risedevtool/src/task/frontend_service.rs b/src/risedevtool/src/task/frontend_service.rs
index df004cba72871..dd0015ac188bd 100644
--- a/src/risedevtool/src/task/frontend_service.rs
+++ b/src/risedevtool/src/task/frontend_service.rs
@@ -63,7 +63,7 @@ impl FrontendService {
                 config.listen_address, config.health_check_port
             ))
             .arg("--metrics-level")
-            .arg("1");
+            .arg("info");
 
         let provide_meta_node = config.provide_meta_node.as_ref().unwrap();
         if provide_meta_node.is_empty() {
diff --git a/src/risedevtool/src/task/utils.rs b/src/risedevtool/src/task/utils.rs
index 0926818c40c42..dbb52aaa5e644 100644
--- a/src/risedevtool/src/task/utils.rs
+++ b/src/risedevtool/src/task/utils.rs
@@ -119,27 +119,28 @@ pub fn add_hummock_backend(
         ([], [], [opendal]) => {
             if opendal.engine == "hdfs"{
                 cmd.arg("--state-store")
-                .arg(format!("hummock+hdfs://{}@{}", opendal.namenode, opendal.root));
+                .arg(format!("hummock+hdfs://{}", opendal.namenode));
             }
             else if opendal.engine == "gcs"{
                 cmd.arg("--state-store")
-                .arg(format!("hummock+gcs://{}@{}", opendal.bucket, opendal.root));
+                .arg(format!("hummock+gcs://{}", opendal.bucket));
             }
             else if opendal.engine == "oss"{
                 cmd.arg("--state-store")
-                .arg(format!("hummock+oss://{}@{}", opendal.bucket, opendal.root));
+                .arg(format!("hummock+oss://{}", opendal.bucket));
             }
             else if opendal.engine == "webhdfs"{
                 cmd.arg("--state-store")
-                .arg(format!("hummock+webhdfs://{}@{}", opendal.namenode, opendal.root));
+                .arg(format!("hummock+webhdfs://{}", opendal.namenode));
             }
             else if opendal.engine == "azblob"{
                 cmd.arg("--state-store")
-                .arg(format!("hummock+azblob://{}@{}", opendal.bucket, opendal.root));
+                .arg(format!("hummock+azblob://{}", opendal.bucket));
             }
             else if opendal.engine == "fs"{
+                println!("using fs engine xxxx");
                 cmd.arg("--state-store")
-                .arg(format!("hummock+fs://{}@{}", opendal.namenode, opendal.root));
+                .arg("hummock+fs://");
             }
             else{
                 unimplemented!()
diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml
index f830e25641164..97225211d8ce9 100644
--- a/src/rpc_client/Cargo.toml
+++ b/src/rpc_client/Cargo.toml
@@ -41,7 +41,7 @@ tokio-stream = "0.1"
 tonic = { workspace = true }
 tower = "0.4"
 tracing = "0.1"
-url = "2.4.0"
+url = "2.4.1"
 
 [target.'cfg(not(madsim))'.dependencies]
 moka = { version = "0.11", features = ["future"] }
diff --git a/src/rpc_client/src/compute_client.rs b/src/rpc_client/src/compute_client.rs
index 83a4be77d3688..aac767570052e 100644
--- a/src/rpc_client/src/compute_client.rs
+++ b/src/rpc_client/src/compute_client.rs
@@ -18,6 +18,7 @@ use std::time::Duration;
 use async_trait::async_trait;
 use futures::StreamExt;
 use risingwave_common::config::{MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE};
+use risingwave_common::monitor::connection::{EndpointExt, TcpConfig};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::tracing::TracingContext;
 use risingwave_pb::batch_plan::{PlanFragment, TaskId, TaskOutputId};
@@ -58,9 +59,14 @@ impl ComputeClient {
         let channel = Endpoint::from_shared(format!("http://{}", &addr))?
             .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE)
             .initial_stream_window_size(STREAM_WINDOW_SIZE)
-            .tcp_nodelay(true)
             .connect_timeout(Duration::from_secs(5))
-            .connect()
+            .monitored_connect(
+                "grpc-compute-client",
+                TcpConfig {
+                    tcp_nodelay: true,
+                    keepalive_duration: None,
+                },
+            )
             .await?;
         Ok(Self::with_channel(addr, channel))
     }
diff --git a/src/rpc_client/src/connector_client.rs b/src/rpc_client/src/connector_client.rs
index 1097e4aee84b7..12386628e5516 100644
--- a/src/rpc_client/src/connector_client.rs
+++ b/src/rpc_client/src/connector_client.rs
@@ -18,6 +18,7 @@ use std::time::Duration;
 
 use anyhow::anyhow;
 use risingwave_common::config::{MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE};
+use risingwave_common::monitor::connection::{EndpointExt, TcpConfig};
 use risingwave_pb::connector_service::connector_service_client::ConnectorServiceClient;
 use risingwave_pb::connector_service::sink_coordinator_stream_request::{
     CommitMetadata, StartCoordinator,
@@ -159,7 +160,6 @@ impl ConnectorClient {
             })?
             .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE)
             .initial_stream_window_size(STREAM_WINDOW_SIZE)
-            .tcp_nodelay(true)
             .connect_timeout(Duration::from_secs(5));
 
         let channel = {
@@ -169,7 +169,13 @@ impl ConnectorClient {
             }
             #[cfg(not(madsim))]
             {
-                endpoint.connect_lazy()
+                endpoint.monitored_connect_lazy(
+                    "grpc-connector-client",
+                    TcpConfig {
+                        tcp_nodelay: true,
+                        keepalive_duration: None,
+                    },
+                )
             }
         };
         Ok(Self {
diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs
index 5f47bb06e69c5..2b87ae995a564 100644
--- a/src/rpc_client/src/meta_client.rs
+++ b/src/rpc_client/src/meta_client.rs
@@ -735,16 +735,16 @@ impl MetaClient {
         Ok(resp.states)
     }
 
-    pub async fn pause(&self) -> Result<()> {
+    pub async fn pause(&self) -> Result<PauseResponse> {
         let request = PauseRequest {};
-        let _resp = self.inner.pause(request).await?;
-        Ok(())
+        let resp = self.inner.pause(request).await?;
+        Ok(resp)
     }
 
-    pub async fn resume(&self) -> Result<()> {
+    pub async fn resume(&self) -> Result<ResumeResponse> {
         let request = ResumeRequest {};
-        let _resp = self.inner.resume(request).await?;
-        Ok(())
+        let resp = self.inner.resume(request).await?;
+        Ok(resp)
     }
 
     pub async fn get_cluster_info(&self) -> Result<GetClusterInfoResponse> {
@@ -1307,7 +1307,7 @@ impl GrpcMetaClientCore {
 
 /// Client to meta server. Cloning the instance is lightweight.
 ///
-/// It is a wrapper of tonic client. See [`rpc_client_method_impl`].
+/// It is a wrapper of tonic client. See [`crate::rpc_client_method_impl`].
 #[derive(Debug, Clone)]
 struct GrpcMetaClient {
     member_monitor_event_sender: mpsc::Sender<Sender<Result<()>>>,
@@ -1645,6 +1645,8 @@ macro_rules! for_all_meta_rpc {
             ,{ cluster_client, list_all_nodes, ListAllNodesRequest, ListAllNodesResponse }
             ,{ heartbeat_client, heartbeat, HeartbeatRequest, HeartbeatResponse }
             ,{ stream_client, flush, FlushRequest, FlushResponse }
+            ,{ stream_client, pause, PauseRequest, PauseResponse }
+            ,{ stream_client, resume, ResumeRequest, ResumeResponse }
             ,{ stream_client, cancel_creating_jobs, CancelCreatingJobsRequest, CancelCreatingJobsResponse }
             ,{ stream_client, list_table_fragments, ListTableFragmentsRequest, ListTableFragmentsResponse }
             ,{ stream_client, list_table_fragment_states, ListTableFragmentStatesRequest, ListTableFragmentStatesResponse }
@@ -1710,8 +1712,6 @@ macro_rules! for_all_meta_rpc {
             ,{ user_client, drop_user, DropUserRequest, DropUserResponse }
             ,{ user_client, grant_privilege, GrantPrivilegeRequest, GrantPrivilegeResponse }
             ,{ user_client, revoke_privilege, RevokePrivilegeRequest, RevokePrivilegeResponse }
-            ,{ scale_client, pause, PauseRequest, PauseResponse }
-            ,{ scale_client, resume, ResumeRequest, ResumeResponse }
             ,{ scale_client, get_cluster_info, GetClusterInfoRequest, GetClusterInfoResponse }
             ,{ scale_client, reschedule, RescheduleRequest, RescheduleResponse }
             ,{ scale_client, get_reschedule_plan, GetReschedulePlanRequest, GetReschedulePlanResponse }
diff --git a/src/rpc_client/src/stream_client.rs b/src/rpc_client/src/stream_client.rs
index c56b8848173cf..09a4c607043eb 100644
--- a/src/rpc_client/src/stream_client.rs
+++ b/src/rpc_client/src/stream_client.rs
@@ -17,6 +17,7 @@ use std::time::Duration;
 
 use async_trait::async_trait;
 use risingwave_common::config::MAX_CONNECTION_WINDOW_SIZE;
+use risingwave_common::monitor::connection::{EndpointExt, TcpConfig};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_pb::stream_service::stream_service_client::StreamServiceClient;
 use risingwave_pb::stream_service::*;
@@ -41,11 +42,19 @@ impl StreamClient {
         let channel = Endpoint::from_shared(format!("http://{}", &host_addr))?
             .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE)
             .connect_timeout(Duration::from_secs(5))
-            .connect()
+            .monitored_connect(
+                "grpc-stream-client",
+                TcpConfig {
+                    tcp_nodelay: true,
+                    keepalive_duration: None,
+                },
+            )
             .await?
             .tracing_injected();
 
-        Ok(Self(StreamServiceClient::new(channel)))
+        Ok(Self(
+            StreamServiceClient::new(channel).max_decoding_message_size(usize::MAX),
+        ))
     }
 }
 
diff --git a/src/source/src/source_desc.rs b/src/source/src/source_desc.rs
index 2276bdf02ebaf..4d4b9f9cb5b80 100644
--- a/src/source/src/source_desc.rs
+++ b/src/source/src/source_desc.rs
@@ -20,7 +20,9 @@ use risingwave_common::error::ErrorCode::ProtocolError;
 use risingwave_common::error::{Result, RwError};
 use risingwave_connector::parser::SpecificParserConfig;
 use risingwave_connector::source::monitor::SourceMetrics;
-use risingwave_connector::source::{SourceColumnDesc, SourceEncode, SourceFormat, SourceStruct};
+use risingwave_connector::source::{
+    SourceColumnDesc, SourceColumnType, SourceEncode, SourceFormat, SourceStruct,
+};
 use risingwave_connector::ConnectorParams;
 use risingwave_pb::catalog::PbStreamSourceInfo;
 use risingwave_pb::plan_common::{PbColumnCatalog, PbEncodeType, PbFormatType, RowFormatType};
@@ -91,7 +93,7 @@ impl SourceDescBuilder {
             .map(|c| SourceColumnDesc::from(&ColumnDesc::from(c.column_desc.as_ref().unwrap())))
             .collect();
         if let Some(row_id_index) = self.row_id_index {
-            columns[row_id_index].is_row_id = true;
+            columns[row_id_index].column_type = SourceColumnType::RowId;
         }
         for pk_index in &self.pk_indices {
             columns[*pk_index].is_pk = true;
diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs
index 86d3ea3bbc09e..ecae5a9663a88 100644
--- a/src/sqlparser/src/ast/mod.rs
+++ b/src/sqlparser/src/ast/mod.rs
@@ -354,32 +354,32 @@ pub enum Expr {
         timestamp: Box<Expr>,
         time_zone: String,
     },
-    /// EXTRACT(DateTimeField FROM <expr>)
+    /// `EXTRACT(DateTimeField FROM <expr>)`
     Extract {
         field: String,
         expr: Box<Expr>,
     },
-    /// SUBSTRING(<expr> [FROM <expr>] [FOR <expr>])
+    /// `SUBSTRING(<expr> [FROM <expr>] [FOR <expr>])`
     Substring {
         expr: Box<Expr>,
         substring_from: Option<Box<Expr>>,
         substring_for: Option<Box<Expr>>,
     },
-    /// POSITION(<expr> IN <expr>)
+    /// `POSITION(<expr> IN <expr>)`
     Position {
         substring: Box<Expr>,
         string: Box<Expr>,
     },
-    /// OVERLAY(<expr> PLACING <expr> FROM <expr> [ FOR <expr> ])
+    /// `OVERLAY(<expr> PLACING <expr> FROM <expr> [ FOR <expr> ])`
     Overlay {
         expr: Box<Expr>,
         new_substring: Box<Expr>,
         start: Box<Expr>,
         count: Option<Box<Expr>>,
     },
-    /// TRIM([BOTH | LEADING | TRAILING] [<expr>] FROM <expr>)\
+    /// `TRIM([BOTH | LEADING | TRAILING] [<expr>] FROM <expr>)`\
     /// Or\
-    /// TRIM([BOTH | LEADING | TRAILING] [FROM] <expr> [, <expr>])
+    /// `TRIM([BOTH | LEADING | TRAILING] [FROM] <expr> [, <expr>])`
     Trim {
         expr: Box<Expr>,
         // ([BOTH | LEADING | TRAILING], <expr>)
@@ -1104,7 +1104,7 @@ pub enum Statement {
     CreateConnection { stmt: CreateConnectionStatement },
     /// CREATE FUNCTION
     ///
-    /// Postgres: https://www.postgresql.org/docs/15/sql-createfunction.html
+    /// Postgres: <https://www.postgresql.org/docs/15/sql-createfunction.html>
     CreateFunction {
         or_replace: bool,
         temporary: bool,
@@ -1186,7 +1186,7 @@ pub enum Statement {
         /// `CASCADE` or `RESTRICT`
         option: Option<ReferentialAction>,
     },
-    /// SET <variable>
+    /// `SET <variable>`
     ///
     /// Note: this is not a standard SQL statement, but it is supported by at
     /// least MySQL and PostgreSQL. Not all MySQL-specific syntactic forms are
@@ -1196,7 +1196,7 @@ pub enum Statement {
         variable: Ident,
         value: Vec<SetVariableValue>,
     },
-    /// SHOW <variable>
+    /// `SHOW <variable>`
     ///
     /// Note: this is a PostgreSQL-specific statement.
     ShowVariable { variable: Vec<Ident> },
diff --git a/src/sqlparser/src/ast/operator.rs b/src/sqlparser/src/ast/operator.rs
index d630c5c2d7fac..ad084a59425b0 100644
--- a/src/sqlparser/src/ast/operator.rs
+++ b/src/sqlparser/src/ast/operator.rs
@@ -149,7 +149,7 @@ impl fmt::Display for BinaryOperator {
 }
 
 /// Qualified custom operator
-/// https://www.postgresql.org/docs/15/sql-expressions.html#SQL-EXPRESSIONS-OPERATOR-CALLS
+/// <https://www.postgresql.org/docs/15/sql-expressions.html#SQL-EXPRESSIONS-OPERATOR-CALLS>
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct QualifiedOperator {
diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs
index feca4986badd1..1917a04e85b9d 100644
--- a/src/sqlparser/src/parser.rs
+++ b/src/sqlparser/src/parser.rs
@@ -133,7 +133,7 @@ impl std::error::Error for ParserError {}
 type ColumnsDefTuple = (Vec<ColumnDef>, Vec<TableConstraint>, Vec<SourceWatermark>);
 
 /// Reference:
-/// https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-PRECEDENCE
+/// <https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-PRECEDENCE>
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Precedence {
     Zero = 0,
@@ -549,7 +549,7 @@ impl Parser {
         }));
 
         let token = self.next_token();
-        let expr = match token.token {
+        let expr = match token.token.clone() {
             Token::Word(w) => match w.keyword {
                 Keyword::TRUE | Keyword::FALSE | Keyword::NULL => {
                     self.prev_token();
@@ -593,7 +593,7 @@ impl Parser {
                     })
                 }
                 k if keywords::RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(&k) => {
-                    parser_err!(format!("syntax error at or near \"{w}\""))
+                    parser_err!(format!("syntax error at or near {token}"))
                 }
                 // Here `w` is a word, check if it's a part of a multi-part
                 // identifier, a function call, or a simple identifier:
@@ -1093,7 +1093,7 @@ impl Parser {
         })
     }
 
-    /// POSITION(<expr> IN <expr>)
+    /// `POSITION(<expr> IN <expr>)`
     pub fn parse_position_expr(&mut self) -> Result<Expr, ParserError> {
         self.expect_token(&Token::LParen)?;
 
@@ -1112,7 +1112,7 @@ impl Parser {
         })
     }
 
-    /// OVERLAY(<expr> PLACING <expr> FROM <expr> [ FOR <expr> ])
+    /// `OVERLAY(<expr> PLACING <expr> FROM <expr> [ FOR <expr> ])`
     pub fn parse_overlay_expr(&mut self) -> Result<Expr, ParserError> {
         self.expect_token(&Token::LParen)?;
 
@@ -1139,8 +1139,8 @@ impl Parser {
         })
     }
 
-    /// TRIM ([WHERE] ['text'] FROM 'text')\
-    /// TRIM ([WHERE] [FROM] 'text' [, 'text'])
+    /// `TRIM ([WHERE] ['text'] FROM 'text')`\
+    /// `TRIM ([WHERE] [FROM] 'text' [, 'text'])`
     pub fn parse_trim_expr(&mut self) -> Result<Expr, ParserError> {
         self.expect_token(&Token::LParen)?;
         let mut trim_where = None;
@@ -1232,7 +1232,7 @@ impl Parser {
                 // for keyword 'array'
                 self.prev_token();
             }
-            parser_err!(format!("syntax error at or near '{}'", self.peek_token()))?
+            parser_err!(format!("syntax error at or near {}", self.peek_token()))?
         } else {
             Ok(())
         }
@@ -1540,8 +1540,8 @@ impl Parser {
         }
     }
 
-    /// We parse both array[1,9][1], array[1,9][1:2], array[1,9][:2], array[1,9][1:] and
-    /// array[1,9][:] in this function.
+    /// We parse both `array[1,9][1]`, `array[1,9][1:2]`, `array[1,9][:2]`, `array[1,9][1:]` and
+    /// `array[1,9][:]` in this function.
     pub fn parse_array_index(&mut self, expr: Expr) -> Result<Expr, ParserError> {
         let new_expr = match self.peek_token().token {
             Token::Colon => {
@@ -3443,10 +3443,10 @@ impl Parser {
     /// Parse a simple one-word identifier (possibly quoted, possibly a non-reserved keyword)
     pub fn parse_identifier_non_reserved(&mut self) -> Result<Ident, ParserError> {
         let token = self.next_token();
-        match token.token {
+        match token.token.clone() {
             Token::Word(w) => {
                 match keywords::RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(&w.keyword) {
-                    true => parser_err!(format!("syntax error at or near \"{w}\"")),
+                    true => parser_err!(format!("syntax error at or near {token}")),
                     false => Ok(w.to_ident()?),
                 }
             }
diff --git a/src/sqlparser/tests/sqlparser_common.rs b/src/sqlparser/tests/sqlparser_common.rs
index ce3ec095ce283..0fc2f3c2530f7 100644
--- a/src/sqlparser/tests/sqlparser_common.rs
+++ b/src/sqlparser/tests/sqlparser_common.rs
@@ -253,7 +253,8 @@ fn parse_select_all() {
 #[test]
 fn parse_select_all_distinct() {
     let result = parse_sql_statements("SELECT ALL DISTINCT name FROM customer");
-    assert!(format!("{}", result.unwrap_err()).contains("syntax error at or near \"DISTINCT\""));
+    assert!(format!("{}", result.unwrap_err())
+        .contains("syntax error at or near DISTINCT at line:1, column:20"));
 }
 
 #[test]
diff --git a/src/sqlparser/tests/sqlparser_postgres.rs b/src/sqlparser/tests/sqlparser_postgres.rs
index 4e45f93a8c8ad..d94f1b06b166b 100644
--- a/src/sqlparser/tests/sqlparser_postgres.rs
+++ b/src/sqlparser/tests/sqlparser_postgres.rs
@@ -1047,7 +1047,7 @@ fn parse_array() {
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near '[ at line:1, column:28'".to_string()
+            "syntax error at or near [ at line:1, column:28".to_string()
         ))
     );
 
@@ -1055,7 +1055,7 @@ fn parse_array() {
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near '[ at line:1, column:24'".to_string()
+            "syntax error at or near [ at line:1, column:24".to_string()
         ))
     );
 
@@ -1063,7 +1063,7 @@ fn parse_array() {
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near 'ARRAY at line:1, column:27'".to_string()
+            "syntax error at or near ARRAY at line:1, column:27".to_string()
         ))
     );
 
@@ -1071,7 +1071,7 @@ fn parse_array() {
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near 'ARRAY at line:1, column:23'".to_string()
+            "syntax error at or near ARRAY at line:1, column:23".to_string()
         ))
     );
 
diff --git a/src/sqlparser/tests/testdata/create.yaml b/src/sqlparser/tests/testdata/create.yaml
index ab5822fc2bc09..92bdabc83048c 100644
--- a/src/sqlparser/tests/testdata/create.yaml
+++ b/src/sqlparser/tests/testdata/create.yaml
@@ -35,7 +35,7 @@
 - input: CREATE TABLE T (a STRUCT<v1 INT>)
   formatted_sql: CREATE TABLE T (a STRUCT<v1 INT>)
 - input: CREATE TABLE T (FULL INT)
-  error_msg: 'sql parser error: syntax error at or near "FULL"'
+  error_msg: 'sql parser error: syntax error at or near FULL at line:1, column:21'
 - input: CREATE TABLE T ("FULL" INT)
   formatted_sql: CREATE TABLE T ("FULL" INT)
 - input: CREATE USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml
index b98b9b6ff4fb2..bbbb5a72bbdab 100644
--- a/src/sqlparser/tests/testdata/select.yaml
+++ b/src/sqlparser/tests/testdata/select.yaml
@@ -71,9 +71,9 @@
     sql parser error: Expected ), found: minutes at line:1, column:62
     Near "(t, x, interval '10'"
 - input: SELECT 1, FROM t
-  error_msg: 'sql parser error: syntax error at or near "FROM"'
+  error_msg: 'sql parser error: syntax error at or near FROM at line:1, column:15'
 - input: SELECT 1, WHERE true
-  error_msg: 'sql parser error: syntax error at or near "WHERE"'
+  error_msg: 'sql parser error: syntax error at or near WHERE at line:1, column:16'
 - input: SELECT timestamp with time zone '2022-10-01 12:00:00Z' AT TIME ZONE 'US/Pacific'
   formatted_sql: SELECT TIMESTAMP WITH TIME ZONE '2022-10-01 12:00:00Z' AT TIME ZONE 'US/Pacific'
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(AtTimeZone { timestamp: TypedString { data_type: Timestamp(true), value: "2022-10-01 12:00:00Z" }, time_zone: "US/Pacific" })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml
index 20d6d7d1e9957..1b0c42502e995 100644
--- a/src/storage/Cargo.toml
+++ b/src/storage/Cargo.toml
@@ -26,8 +26,7 @@ dyn-clone = "1.0.13"
 either = "1"
 enum-as-inner = "0.6"
 fail = "0.5"
-# FIXME: change back to mrcroxx/foyer later
-foyer = { git = "https://github.com/xxchan/foyer", rev = "fc5e4cb" }
+foyer = { git = "https://github.com/mrcroxx/foyer", rev = "2b8907c" }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
 hex = "0.4"
@@ -54,7 +53,6 @@ scopeguard = "1"
 sled = "0.34.7"
 spin = "0.9"
 sync-point = { path = "../utils/sync-point" }
-sysinfo = { version = "0.29", default-features = false }
 tempfile = "3"
 thiserror = "1"
 # tikv-client = { git = "https://github.com/tikv/client-rust", rev = "5714b2", optional = true }
@@ -71,7 +69,7 @@ tokio-retry = "0.3"
 tracing = "0.1"
 tracing-futures = { version = "0.2", features = ["futures-03"] }
 xorf = "0.8.1"
-xxhash-rust = { version = "0.8.6", features = ["xxh32", "xxh64"] }
+xxhash-rust = { version = "0.8.7", features = ["xxh32", "xxh64"] }
 zstd = { version = "0.12", default-features = false }
 
 [target.'cfg(target_os = "linux")'.dependencies]
diff --git a/src/storage/compactor/src/lib.rs b/src/storage/compactor/src/lib.rs
index 93aee41c06e1a..8043cb5d2214d 100644
--- a/src/storage/compactor/src/lib.rs
+++ b/src/storage/compactor/src/lib.rs
@@ -18,7 +18,7 @@ mod server;
 mod telemetry;
 
 use clap::Parser;
-use risingwave_common::config::{AsyncStackTraceOption, OverrideConfig};
+use risingwave_common::config::{AsyncStackTraceOption, MetricLevel, OverrideConfig};
 
 use crate::server::compactor_serve;
 
@@ -66,11 +66,9 @@ pub struct CompactorOpts {
     pub config_path: String,
 
     /// Used for control the metrics level, similar to log level.
-    /// 0 = close metrics
-    /// >0 = open metrics
     #[clap(long, env = "RW_METRICS_LEVEL")]
     #[override_opts(path = server.metrics_level)]
-    pub metrics_level: Option<u32>,
+    pub metrics_level: Option<MetricLevel>,
 
     /// Enable async stack tracing through `await-tree` for risectl.
     #[clap(long, env = "RW_ASYNC_STACK_TRACE", value_enum)]
diff --git a/src/storage/compactor/src/server.rs b/src/storage/compactor/src/server.rs
index 8bffcd0b168b7..029705ef3c204 100644
--- a/src/storage/compactor/src/server.rs
+++ b/src/storage/compactor/src/server.rs
@@ -19,8 +19,9 @@ use std::time::Duration;
 
 use parking_lot::RwLock;
 use risingwave_common::config::{
-    extract_storage_memory_config, load_config, AsyncStackTraceOption,
+    extract_storage_memory_config, load_config, AsyncStackTraceOption, MetricLevel,
 };
+use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
 use risingwave_common::telemetry::manager::TelemetryManager;
 use risingwave_common::telemetry::telemetry_env_enabled;
@@ -30,12 +31,14 @@ use risingwave_common::{GIT_SHA, RW_VERSION};
 use risingwave_common_service::metrics_manager::MetricsManager;
 use risingwave_common_service::observer_manager::ObserverManager;
 use risingwave_object_store::object::object_metrics::GLOBAL_OBJECT_STORE_METRICS;
-use risingwave_object_store::object::parse_remote_object_store_with_config;
+use risingwave_object_store::object::parse_remote_object_store;
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::compactor::compactor_service_server::CompactorServiceServer;
 use risingwave_pb::monitor_service::monitor_service_server::MonitorServiceServer;
 use risingwave_rpc_client::MetaClient;
-use risingwave_storage::filter_key_extractor::{FilterKeyExtractorManager, RemoteTableAccessor};
+use risingwave_storage::filter_key_extractor::{
+    FilterKeyExtractorManager, RemoteTableAccessor, RpcFilterKeyExtractorManager,
+};
 use risingwave_storage::hummock::compactor::{CompactionExecutor, CompactorContext};
 use risingwave_storage::hummock::hummock_meta_client::MonitoredHummockMetaClient;
 use risingwave_storage::hummock::{
@@ -131,13 +134,12 @@ pub async fn compactor_serve(
         assert!(compactor_memory_limit_bytes > min_compactor_memory_limit_bytes * 2);
     }
 
-    let mut object_store = parse_remote_object_store_with_config(
+    let mut object_store = parse_remote_object_store(
         state_store_url
             .strip_prefix("hummock+")
             .expect("object store must be hummock for compactor server"),
         object_metrics,
         "Hummock",
-        Some(Arc::new(config.storage.clone())),
     )
     .await;
     object_store.set_opts(
@@ -156,7 +158,7 @@ pub async fn compactor_serve(
 
     let telemetry_enabled = system_params_reader.telemetry_enabled();
 
-    let filter_key_extractor_manager = Arc::new(FilterKeyExtractorManager::new(Box::new(
+    let filter_key_extractor_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new(
         RemoteTableAccessor::new(meta_client.clone()),
     )));
     let system_params_manager = Arc::new(LocalSystemParamsManager::new(system_params_reader));
@@ -192,29 +194,34 @@ pub async fn compactor_serve(
     };
     let await_tree_reg =
         await_tree_config.map(|c| Arc::new(RwLock::new(await_tree::Registry::new(c))));
-    let compactor_context = Arc::new(CompactorContext {
+    let compactor_context = CompactorContext {
         storage_opts,
-        hummock_meta_client: hummock_meta_client.clone(),
         sstable_store: sstable_store.clone(),
         compactor_metrics,
         is_share_buffer_compact: false,
         compaction_executor: Arc::new(CompactionExecutor::new(
             opts.compaction_worker_threads_number,
         )),
-        filter_key_extractor_manager: filter_key_extractor_manager.clone(),
+        filter_key_extractor_manager: FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+            filter_key_extractor_manager.clone(),
+        ),
         memory_limiter,
-        sstable_object_id_manager: sstable_object_id_manager.clone(),
+
         task_progress_manager: Default::default(),
         await_tree_reg: await_tree_reg.clone(),
         running_task_count: Arc::new(AtomicU32::new(0)),
-    });
+    };
     let mut sub_tasks = vec![
         MetaClient::start_heartbeat_loop(
             meta_client.clone(),
             Duration::from_millis(config.server.heartbeat_interval_ms as u64),
-            vec![sstable_object_id_manager],
+            vec![sstable_object_id_manager.clone()],
+        ),
+        risingwave_storage::hummock::compactor::start_compactor(
+            compactor_context.clone(),
+            hummock_meta_client.clone(),
+            sstable_object_id_manager.clone(),
         ),
-        risingwave_storage::hummock::compactor::start_compactor(compactor_context.clone()),
     ];
 
     let telemetry_manager = TelemetryManager::new(
@@ -240,28 +247,35 @@ pub async fn compactor_serve(
         tonic::transport::Server::builder()
             .add_service(CompactorServiceServer::new(compactor_srv))
             .add_service(MonitorServiceServer::new(monitor_srv))
-            .serve_with_shutdown(listen_addr, async move {
-                tokio::select! {
-                    _ = tokio::signal::ctrl_c() => {},
-                    _ = &mut shutdown_recv => {
-                        for (join_handle, shutdown_sender) in sub_tasks {
-                            if let Err(err) = shutdown_sender.send(()) {
-                                tracing::warn!("Failed to send shutdown: {:?}", err);
-                                continue;
-                            }
-                            if let Err(err) = join_handle.await {
-                                tracing::warn!("Failed to join shutdown: {:?}", err);
+            .monitored_serve_with_shutdown(
+                listen_addr,
+                "grpc-compactor-node-service",
+                TcpConfig {
+                    tcp_nodelay: true,
+                    keepalive_duration: None,
+                },
+                async move {
+                    tokio::select! {
+                        _ = tokio::signal::ctrl_c() => {},
+                        _ = &mut shutdown_recv => {
+                            for (join_handle, shutdown_sender) in sub_tasks {
+                                if let Err(err) = shutdown_sender.send(()) {
+                                    tracing::warn!("Failed to send shutdown: {:?}", err);
+                                    continue;
+                                }
+                                if let Err(err) = join_handle.await {
+                                    tracing::warn!("Failed to join shutdown: {:?}", err);
+                                }
                             }
-                        }
-                    },
-                }
-            })
+                        },
+                    }
+                },
+            )
             .await
-            .unwrap();
     });
 
     // Boot metrics service.
-    if config.server.metrics_level > 0 {
+    if config.server.metrics_level > MetricLevel::Disabled {
         MetricsManager::boot_metrics_service(opts.prometheus_listener_addr.clone());
     }
 
diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
index fc7eda084bd15..1193877a14c9b 100644
--- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
+++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::cmp::Ordering;
+use std::collections::hash_map::Entry;
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 
 use itertools::Itertools;
@@ -23,12 +24,13 @@ use risingwave_pb::hummock::hummock_version_delta::GroupDeltas;
 use risingwave_pb::hummock::{
     CompactionConfig, CompatibilityVersion, GroupConstruct, GroupDestroy, GroupMetaChange,
     GroupTableChange, HummockVersion, HummockVersionDelta, Level, LevelType, OverlappingLevel,
-    SstableInfo,
+    PbLevelType, SstableInfo,
 };
 use tracing::warn;
 
 use super::StateTableId;
 use crate::compaction_group::StaticCompactionGroupId;
+use crate::key_range::KeyRangeCommon;
 use crate::prost_key_range::KeyRangeExt;
 use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId};
 
@@ -348,7 +350,7 @@ impl HummockVersionUpdateExt for HummockVersion {
                 }
             }
         }
-        for (z, level) in parent_levels.levels.iter_mut().enumerate() {
+        for (idx, level) in parent_levels.levels.iter_mut().enumerate() {
             let insert_table_infos = split_sst_info_for_level(
                 &member_table_ids,
                 allow_trivial_split,
@@ -356,21 +358,23 @@ impl HummockVersionUpdateExt for HummockVersion {
                 &mut split_id_vers,
                 &mut new_sst_id,
             );
-            cur_levels.levels[z].total_file_size += insert_table_infos
+            cur_levels.levels[idx].total_file_size += insert_table_infos
                 .iter()
                 .map(|sst| sst.file_size)
                 .sum::<u64>();
-            cur_levels.levels[z].uncompressed_file_size += insert_table_infos
+            cur_levels.levels[idx].uncompressed_file_size += insert_table_infos
                 .iter()
                 .map(|sst| sst.uncompressed_file_size)
                 .sum::<u64>();
-            cur_levels.levels[z].table_infos.extend(insert_table_infos);
-            cur_levels.levels[z].table_infos.sort_by(|sst1, sst2| {
+            cur_levels.levels[idx]
+                .table_infos
+                .extend(insert_table_infos);
+            cur_levels.levels[idx].table_infos.sort_by(|sst1, sst2| {
                 let a = sst1.key_range.as_ref().unwrap();
                 let b = sst2.key_range.as_ref().unwrap();
                 a.compare(b)
             });
-            assert!(can_concat(&cur_levels.levels[z].table_infos));
+            assert!(can_concat(&cur_levels.levels[idx].table_infos));
             level
                 .table_infos
                 .extract_if(|sst_info| sst_info.table_ids.is_empty())
@@ -1057,6 +1061,145 @@ pub fn object_size_map(version: &HummockVersion) -> HashMap<HummockSstableObject
         .collect()
 }
 
+/// Verify the validity of a `HummockVersion` and return a list of violations if any.
+/// Currently this method is only used by risectl validate-version.
+pub fn validate_version(version: &HummockVersion) -> Vec<String> {
+    let mut res = Vec::new();
+
+    // Ensure safe_epoch <= max_committed_epoch
+    if version.safe_epoch > version.max_committed_epoch {
+        res.push(format!(
+            "VERSION: safe_epoch {} > max_committed_epoch {}",
+            version.safe_epoch, version.max_committed_epoch
+        ));
+    }
+
+    let mut table_to_group = HashMap::new();
+    // Ensure each table maps to only one compaction group
+    for (group_id, levels) in &version.levels {
+        // Ensure compaction group id matches
+        if levels.group_id != *group_id {
+            res.push(format!(
+                "GROUP {}: inconsistent group id {} in Levels",
+                group_id, levels.group_id
+            ));
+        }
+
+        // Ensure table id is sorted
+        if !levels.member_table_ids.is_sorted() {
+            res.push(format!(
+                "GROUP {}: memtable_table_ids is not sorted: {:?}",
+                group_id, levels.member_table_ids
+            ));
+        }
+
+        // Ensure table id is unique
+        for table_id in &levels.member_table_ids {
+            match table_to_group.entry(table_id) {
+                Entry::Occupied(e) => {
+                    res.push(format!(
+                        "GROUP {}: Duplicated table_id {}. First found in group {}",
+                        group_id,
+                        table_id,
+                        e.get()
+                    ));
+                }
+                Entry::Vacant(e) => {
+                    e.insert(group_id);
+                }
+            }
+        }
+
+        let validate_level = |group: CompactionGroupId,
+                              expected_level_idx: u32,
+                              level: &Level,
+                              res: &mut Vec<String>| {
+            let mut level_identifier = format!("GROUP {} LEVEL {}", group, level.level_idx);
+            if level.level_idx == 0 {
+                level_identifier.push_str(format!("SUBLEVEL {}", level.sub_level_id).as_str());
+                // Ensure sub-level is not empty
+                if level.table_infos.is_empty() {
+                    res.push(format!("{}: empty level", level_identifier));
+                }
+            } else if level.level_type() != PbLevelType::Nonoverlapping {
+                // Ensure non-L0 level is non-overlapping level
+                res.push(format!(
+                    "{}: level type {:?} is not non-overlapping",
+                    level_identifier,
+                    level.level_type()
+                ));
+            }
+
+            // Ensure level idx matches
+            if level.level_idx != expected_level_idx {
+                res.push(format!(
+                    "{}: mismatched level idx {}",
+                    level_identifier, expected_level_idx
+                ));
+            }
+
+            let mut prev_table_info: Option<&SstableInfo> = None;
+            for table_info in &level.table_infos {
+                // Ensure table_ids are sorted and unique
+                if !table_info.table_ids.is_sorted_by(|a, b| {
+                    if a < b {
+                        Some(Ordering::Less)
+                    } else {
+                        Some(Ordering::Greater)
+                    }
+                }) {
+                    res.push(format!(
+                        "{} SST {}: table_ids not sorted",
+                        level_identifier, table_info.object_id
+                    ));
+                }
+
+                // Ensure SSTs in non-overlapping level have non-overlapping key range
+                if level.level_type() == PbLevelType::Nonoverlapping {
+                    if let Some(prev) = prev_table_info.take() {
+                        if prev
+                            .key_range
+                            .as_ref()
+                            .unwrap()
+                            .compare_right_with(&table_info.key_range.as_ref().unwrap().left)
+                            != Ordering::Less
+                        {
+                            res.push(format!(
+                                "{} SST {}: key range should not overlap. prev={:?}, cur={:?}",
+                                level_identifier, table_info.object_id, prev, table_info
+                            ));
+                        }
+                    }
+                    let _ = prev_table_info.insert(table_info);
+                }
+            }
+        };
+
+        if let Some(l0) = &levels.l0 {
+            let mut prev_sub_level_id = u64::MAX;
+            for sub_level in &l0.sub_levels {
+                // Ensure sub_level_id is sorted and unique
+                if sub_level.sub_level_id >= prev_sub_level_id {
+                    res.push(format!(
+                        "GROUP {} LEVEL 0: sub_level_id {} >= prev_sub_level {}",
+                        group_id, sub_level.level_idx, prev_sub_level_id
+                    ));
+                }
+                prev_sub_level_id = sub_level.sub_level_id;
+
+                validate_level(*group_id, 0, sub_level, &mut res);
+            }
+        } else {
+            res.push(format!("GROUP {}: level0 not exist", group_id));
+        }
+
+        for idx in 1..=levels.levels.len() {
+            validate_level(*group_id, idx as u32, levels.get_level(idx), &mut res);
+        }
+    }
+    res
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
diff --git a/src/storage/hummock_sdk/src/key.rs b/src/storage/hummock_sdk/src/key.rs
index 7013b3350ff9d..f58f32dd9aac9 100644
--- a/src/storage/hummock_sdk/src/key.rs
+++ b/src/storage/hummock_sdk/src/key.rs
@@ -397,13 +397,27 @@ impl<T: AsRef<[u8]>> AsRef<[u8]> for TableKey<T> {
     }
 }
 
+impl<T: AsRef<[u8]>> TableKey<T> {
+    pub fn vnode_part(&self) -> VirtualNode {
+        VirtualNode::from_be_bytes(
+            self.0.as_ref()[..VirtualNode::SIZE]
+                .try_into()
+                .expect("slice with incorrect length"),
+        )
+    }
+
+    pub fn key_part(&self) -> &[u8] {
+        &self.0.as_ref()[VirtualNode::SIZE..]
+    }
+}
+
 #[inline]
 pub fn map_table_key_range(range: (Bound<KeyPayloadType>, Bound<KeyPayloadType>)) -> TableKeyRange {
     (range.0.map(TableKey), range.1.map(TableKey))
 }
 
 /// [`UserKey`] is is an internal concept in storage. In the storage interface, user specifies
-/// `table_key` and `table_id` (in [`ReadOptions`] or [`WriteOptions`]) as the input. The storage
+/// `table_key` and `table_id` (in `ReadOptions` or `WriteOptions`) as the input. The storage
 /// will group these two values into one struct for convenient filtering.
 ///
 /// The encoded format is | `table_id` | `table_key` |.
@@ -474,12 +488,7 @@ impl<T: AsRef<[u8]>> UserKey<T> {
     }
 
     pub fn get_vnode_id(&self) -> usize {
-        VirtualNode::from_be_bytes(
-            self.table_key.as_ref()[..VirtualNode::SIZE]
-                .try_into()
-                .expect("slice with incorrect length"),
-        )
-        .to_index()
+        self.table_key.vnode_part().to_index()
     }
 }
 
diff --git a/src/storage/hummock_sdk/src/key_cmp.rs b/src/storage/hummock_sdk/src/key_cmp.rs
index 9c865e57c2636..e0677e2cc499c 100644
--- a/src/storage/hummock_sdk/src/key_cmp.rs
+++ b/src/storage/hummock_sdk/src/key_cmp.rs
@@ -17,7 +17,8 @@ use std::cmp::{self, Ordering};
 use super::key::split_key_epoch;
 use crate::key::UserKey;
 
-/// A comparator for comparing [`FullKey`] and [`UserKey`] with possibly different table key types.
+/// A comparator for comparing [`crate::key::FullKey`] and [`crate::key::UserKey`] with possibly
+/// different table key types.
 pub struct KeyComparator;
 
 impl KeyComparator {
diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs
index b48dc5c298fbb..0fc6735571e4d 100644
--- a/src/storage/hummock_sdk/src/lib.rs
+++ b/src/storage/hummock_sdk/src/lib.rs
@@ -20,6 +20,7 @@
 #![feature(bound_map)]
 #![feature(type_alias_impl_trait)]
 #![feature(impl_trait_in_assoc_type)]
+#![feature(is_sorted)]
 
 mod key_cmp;
 use std::cmp::Ordering;
diff --git a/src/storage/hummock_test/benches/bench_hummock_iter.rs b/src/storage/hummock_test/benches/bench_hummock_iter.rs
index 6a96b7708d081..3bd6738f9f9a2 100644
--- a/src/storage/hummock_test/benches/bench_hummock_iter.rs
+++ b/src/storage/hummock_test/benches/bench_hummock_iter.rs
@@ -20,6 +20,7 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use futures::{pin_mut, TryStreamExt};
 use risingwave_common::cache::CachePriority;
 use risingwave_hummock_test::get_notification_client_for_test;
+use risingwave_hummock_test::local_state_store_test_utils::LocalStateStoreTestExt;
 use risingwave_hummock_test::test_utils::TestIngestBatch;
 use risingwave_meta::hummock::test_utils::setup_compute_env;
 use risingwave_meta::hummock::MockHummockMetaClient;
@@ -78,7 +79,9 @@ fn criterion_benchmark(c: &mut Criterion) {
     });
 
     let epoch = 100;
-    hummock_storage.init(epoch);
+    runtime
+        .block_on(hummock_storage.init_for_test(epoch))
+        .unwrap();
 
     for batch in batches {
         runtime
diff --git a/src/storage/hummock_test/src/bin/replay/main.rs b/src/storage/hummock_test/src/bin/replay/main.rs
index 968f6437a2f43..8ca23d2ba0327 100644
--- a/src/storage/hummock_test/src/bin/replay/main.rs
+++ b/src/storage/hummock_test/src/bin/replay/main.rs
@@ -38,7 +38,7 @@ use risingwave_meta::hummock::test_utils::setup_compute_env;
 use risingwave_meta::hummock::MockHummockMetaClient;
 use risingwave_object_store::object::parse_remote_object_store;
 use risingwave_storage::filter_key_extractor::{
-    FakeRemoteTableAccessor, FilterKeyExtractorManager,
+    FakeRemoteTableAccessor, RpcFilterKeyExtractorManager,
 };
 use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore};
 use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics, ObjectStoreMetrics};
@@ -137,7 +137,7 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result<impl GlobalRepl
         )
     };
 
-    let key_filter_manager = Arc::new(FilterKeyExtractorManager::new(Box::new(
+    let key_filter_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new(
         FakeRemoteTableAccessor {},
     )));
 
diff --git a/src/storage/hummock_test/src/bin/replay/replay_impl.rs b/src/storage/hummock_test/src/bin/replay/replay_impl.rs
index 52dd109ba512e..e96855d75f925 100644
--- a/src/storage/hummock_test/src/bin/replay/replay_impl.rs
+++ b/src/storage/hummock_test/src/bin/replay/replay_impl.rs
@@ -23,11 +23,10 @@ use risingwave_common_service::observer_manager::{Channel, NotificationClient};
 use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_hummock_trace::{
     GlobalReplay, LocalReplay, LocalReplayRead, ReplayItem, ReplayRead, ReplayStateStore,
-    ReplayWrite, Result, TraceError, TracedBytes, TracedNewLocalOptions, TracedReadOptions,
-    TracedSubResp,
+    ReplayWrite, Result, TraceError, TracedBytes, TracedInitOptions, TracedNewLocalOptions,
+    TracedReadOptions, TracedSubResp,
 };
 use risingwave_meta::manager::{MessageStatus, MetaSrvEnv, NotificationManagerRef, WorkerKey};
-use risingwave_meta::storage::{MemStore, MetaStore};
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::meta::subscribe_response::{Info, Operation as RespOperation};
 use risingwave_pb::meta::{SubscribeResponse, SubscribeType};
@@ -38,6 +37,7 @@ use risingwave_storage::store::{
 };
 use risingwave_storage::{StateStore, StateStoreReadIterStream};
 use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver};
+
 pub(crate) struct GlobalReplayIter<S>
 where
     S: StateStoreReadIterStream,
@@ -87,11 +87,11 @@ impl LocalReplayIter {
 
 pub(crate) struct GlobalReplayImpl {
     store: HummockStorage,
-    notifier: NotificationManagerRef<MemStore>,
+    notifier: NotificationManagerRef,
 }
 
 impl GlobalReplayImpl {
-    pub(crate) fn new(store: HummockStorage, notifier: NotificationManagerRef<MemStore>) -> Self {
+    pub(crate) fn new(store: HummockStorage, notifier: NotificationManagerRef) -> Self {
         Self { store, notifier }
     }
 }
@@ -199,8 +199,11 @@ pub(crate) struct LocalReplayImpl(LocalHummockStorage);
 
 #[async_trait::async_trait]
 impl LocalReplay for LocalReplayImpl {
-    fn init(&mut self, epoch: u64) {
-        self.0.init(epoch);
+    async fn init(&mut self, options: TracedInitOptions) -> Result<()> {
+        self.0
+            .init(options.into())
+            .await
+            .map_err(|_| TraceError::Other("init failed"))
     }
 
     fn seal_current_epoch(&mut self, next_epoch: u64) {
@@ -286,16 +289,16 @@ impl ReplayWrite for LocalReplayImpl {
     }
 }
 
-pub struct ReplayNotificationClient<S: MetaStore> {
+pub struct ReplayNotificationClient {
     addr: HostAddr,
-    notification_manager: NotificationManagerRef<S>,
+    notification_manager: NotificationManagerRef,
     first_resp: Box<TracedSubResp>,
 }
 
-impl<S: MetaStore> ReplayNotificationClient<S> {
+impl ReplayNotificationClient {
     pub fn new(
         addr: HostAddr,
-        notification_manager: NotificationManagerRef<S>,
+        notification_manager: NotificationManagerRef,
         first_resp: Box<TracedSubResp>,
     ) -> Self {
         Self {
@@ -307,7 +310,7 @@ impl<S: MetaStore> ReplayNotificationClient<S> {
 }
 
 #[async_trait::async_trait]
-impl<S: MetaStore> NotificationClient for ReplayNotificationClient<S> {
+impl NotificationClient for ReplayNotificationClient {
     type Channel = ReplayChannel<SubscribeResponse>;
 
     async fn subscribe(&self, subscribe_type: SubscribeType) -> RwResult<Self::Channel> {
@@ -330,10 +333,10 @@ impl<S: MetaStore> NotificationClient for ReplayNotificationClient<S> {
 }
 
 pub fn get_replay_notification_client(
-    env: MetaSrvEnv<MemStore>,
+    env: MetaSrvEnv,
     worker_node: WorkerNode,
     first_resp: Box<TracedSubResp>,
-) -> ReplayNotificationClient<MemStore> {
+) -> ReplayNotificationClient {
     ReplayNotificationClient::new(
         worker_node.get_host().unwrap().into(),
         env.notification_manager_ref(),
diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs
index 8662a06ea3b9b..ca5418cbe6a0f 100644
--- a/src/storage/hummock_test/src/compactor_tests.rs
+++ b/src/storage/hummock_test/src/compactor_tests.rs
@@ -39,14 +39,13 @@ pub(crate) mod tests {
         unregister_table_ids_from_compaction_group,
     };
     use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
-    use risingwave_meta::storage::MetaStore;
     use risingwave_pb::common::{HostAddress, WorkerType};
     use risingwave_pb::hummock::{HummockVersion, TableOption};
     use risingwave_pb::meta::add_worker_node_request::Property;
     use risingwave_rpc_client::HummockMetaClient;
     use risingwave_storage::filter_key_extractor::{
-        FilterKeyExtractorImpl, FilterKeyExtractorManagerRef, FixedLengthFilterKeyExtractor,
-        FullKeyFilterKeyExtractor,
+        FilterKeyExtractorImpl, FilterKeyExtractorManager, FilterKeyExtractorManagerRef,
+        FixedLengthFilterKeyExtractor, FullKeyFilterKeyExtractor,
     };
     use risingwave_storage::hummock::compactor::compactor_runner::compact;
     use risingwave_storage::hummock::compactor::{CompactionExecutor, CompactorContext};
@@ -62,12 +61,13 @@ pub(crate) mod tests {
     use risingwave_storage::store::*;
 
     use crate::get_notification_client_for_test;
+    use crate::local_state_store_test_utils::LocalStateStoreTestExt;
     use crate::test_utils::{register_tables_with_id_for_test, TestIngestBatch};
 
-    pub(crate) async fn get_hummock_storage<S: MetaStore>(
+    pub(crate) async fn get_hummock_storage(
         hummock_meta_client: Arc<dyn HummockMetaClient>,
         notification_client: impl NotificationClient,
-        hummock_manager_ref: &HummockManagerRef<S>,
+        hummock_manager_ref: &HummockManagerRef,
         table_id: TableId,
     ) -> HummockStorage {
         let remote_dir = "hummock_001_test".to_string();
@@ -135,7 +135,7 @@ pub(crate) mod tests {
         let mut local = storage.new_local(Default::default()).await;
         // 1. add sstables
         let val = b"0"[..].repeat(value_size);
-        local.init(epochs[0]);
+        local.init_for_test(epochs[0]).await.unwrap();
         for (i, &epoch) in epochs.iter().enumerate() {
             let mut new_val = val.clone();
             new_val.extend_from_slice(&epoch.to_be_bytes());
@@ -166,13 +166,11 @@ pub(crate) mod tests {
 
     fn get_compactor_context_with_filter_key_extractor_manager(
         storage: &HummockStorage,
-        hummock_meta_client: &Arc<dyn HummockMetaClient>,
         filter_key_extractor_manager: FilterKeyExtractorManagerRef,
     ) -> CompactorContext {
         get_compactor_context_with_filter_key_extractor_manager_impl(
             storage.storage_opts().clone(),
             storage.sstable_store(),
-            hummock_meta_client,
             filter_key_extractor_manager,
         )
     }
@@ -180,22 +178,18 @@ pub(crate) mod tests {
     fn get_compactor_context_with_filter_key_extractor_manager_impl(
         options: Arc<StorageOpts>,
         sstable_store: SstableStoreRef,
-        hummock_meta_client: &Arc<dyn HummockMetaClient>,
         filter_key_extractor_manager: FilterKeyExtractorManagerRef,
     ) -> CompactorContext {
         CompactorContext {
-            storage_opts: options.clone(),
+            storage_opts: options,
             sstable_store,
-            hummock_meta_client: hummock_meta_client.clone(),
             compactor_metrics: Arc::new(CompactorMetrics::unused()),
             is_share_buffer_compact: false,
             compaction_executor: Arc::new(CompactionExecutor::new(Some(1))),
             memory_limiter: MemoryLimiter::unlimit(),
-            filter_key_extractor_manager,
-            sstable_object_id_manager: Arc::new(SstableObjectIdManager::new(
-                hummock_meta_client.clone(),
-                options.sstable_id_remote_fetch_number,
-            )),
+            filter_key_extractor_manager: FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                filter_key_extractor_manager,
+            ),
             task_progress_manager: Default::default(),
             await_tree_reg: None,
             running_task_count: Arc::new(AtomicU32::new(0)),
@@ -227,11 +221,25 @@ pub(crate) mod tests {
             Default::default(),
         )
         .await;
+
+        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
+        {
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                rpc_filter_key_extractor_manager,
+            ) => rpc_filter_key_extractor_manager,
+            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+        };
         let compact_ctx = get_compactor_context_with_filter_key_extractor_manager(
             &storage,
-            &hummock_meta_client,
-            storage.filter_key_extractor_manager().clone(),
+            rpc_filter_key_extractor_manager,
         );
+        let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+            hummock_meta_client.clone(),
+            storage
+                .storage_opts()
+                .clone()
+                .sstable_id_remote_fetch_number,
+        ));
         let worker_node2 = hummock_manager_ref
             .cluster_manager
             .add_worker_node(
@@ -277,8 +285,13 @@ pub(crate) mod tests {
             compact_task.current_epoch_time = 0;
 
             let (_tx, rx) = tokio::sync::oneshot::channel();
-            let (mut result_task, task_stats) =
-                compact(Arc::new(compact_ctx.clone()), compact_task.clone(), rx).await;
+            let (mut result_task, task_stats) = compact(
+                compact_ctx.clone(),
+                compact_task.clone(),
+                rx,
+                Box::new(sstable_object_id_manager.clone()),
+            )
+            .await;
 
             hummock_manager_ref
                 .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
@@ -371,12 +384,25 @@ pub(crate) mod tests {
             Default::default(),
         )
         .await;
+
+        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
+        {
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                rpc_filter_key_extractor_manager,
+            ) => rpc_filter_key_extractor_manager,
+            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+        };
         let compact_ctx = get_compactor_context_with_filter_key_extractor_manager(
             &storage,
-            &hummock_meta_client,
-            storage.filter_key_extractor_manager().clone(),
+            rpc_filter_key_extractor_manager,
         );
-
+        let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+            hummock_meta_client.clone(),
+            storage
+                .storage_opts()
+                .clone()
+                .sstable_id_remote_fetch_number,
+        ));
         // 1. add sstables with 1MB value
         let mut key = BytesMut::default();
         key.put_u16(0);
@@ -421,8 +447,13 @@ pub(crate) mod tests {
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let (mut result_task, task_stats) =
-            compact(Arc::new(compact_ctx), compact_task.clone(), rx).await;
+        let (mut result_task, task_stats) = compact(
+            compact_ctx,
+            compact_task.clone(),
+            rx,
+            Box::new(sstable_object_id_manager.clone()),
+        )
+        .await;
 
         hummock_manager_ref
             .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
@@ -469,17 +500,6 @@ pub(crate) mod tests {
             .unwrap()
             .to_vec();
         assert_eq!(get_val, val);
-
-        // 6. get compact task and there should be none
-        let compact_task = hummock_manager_ref
-            .get_compact_task(
-                StaticCompactionGroupId::StateDefault.into(),
-                &mut default_level_selector(),
-            )
-            .await
-            .unwrap()
-            .unwrap();
-        assert_eq!(6, compact_task.target_level);
     }
 
     pub(crate) async fn flush_and_commit(
@@ -514,7 +534,7 @@ pub(crate) mod tests {
             epoch += 1;
 
             if idx == 0 {
-                local.init(epoch);
+                local.init_for_test(epoch).await.unwrap();
             }
 
             for _ in 0..keys_per_epoch {
@@ -532,19 +552,23 @@ pub(crate) mod tests {
 
     pub(crate) fn prepare_compactor_and_filter(
         storage: &HummockStorage,
-        hummock_meta_client: &Arc<dyn HummockMetaClient>,
         existing_table_id: u32,
     ) -> CompactorContext {
-        let filter_key_extractor_manager = storage.filter_key_extractor_manager().clone();
-        filter_key_extractor_manager.update(
+        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
+        {
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                rpc_filter_key_extractor_manager,
+            ) => rpc_filter_key_extractor_manager,
+            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+        };
+        rpc_filter_key_extractor_manager.update(
             existing_table_id,
             Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
         );
 
         get_compactor_context_with_filter_key_extractor_manager(
             storage,
-            hummock_meta_client,
-            filter_key_extractor_manager,
+            rpc_filter_key_extractor_manager,
         )
     }
 
@@ -636,13 +660,20 @@ pub(crate) mod tests {
             .new_local(NewLocalOptions::for_test(TableId::from(2)))
             .await;
 
-        let filter_key_extractor_manager = global_storage.filter_key_extractor_manager().clone();
-        filter_key_extractor_manager.update(
+        let rpc_filter_key_extractor_manager =
+            match global_storage.filter_key_extractor_manager().clone() {
+                FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                    rpc_filter_key_extractor_manager,
+                ) => rpc_filter_key_extractor_manager,
+                FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+            };
+
+        rpc_filter_key_extractor_manager.update(
             1,
             Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
         );
 
-        filter_key_extractor_manager.update(
+        rpc_filter_key_extractor_manager.update(
             2,
             Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
         );
@@ -650,10 +681,15 @@ pub(crate) mod tests {
         let compact_ctx = get_compactor_context_with_filter_key_extractor_manager_impl(
             global_storage.storage_opts().clone(),
             global_storage.sstable_store(),
-            &hummock_meta_client,
-            filter_key_extractor_manager.clone(),
+            rpc_filter_key_extractor_manager,
         );
-
+        let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+            hummock_meta_client.clone(),
+            global_storage
+                .storage_opts()
+                .clone()
+                .sstable_id_remote_fetch_number,
+        ));
         // 1. add sstables
         let val = Bytes::from(b"0"[..].repeat(1 << 10)); // 1024 Byte value
 
@@ -671,8 +707,8 @@ pub(crate) mod tests {
             epoch += 1;
             let next_epoch = epoch + 1;
             if index == 0 {
-                storage_1.init(epoch);
-                storage_2.init(epoch);
+                storage_1.init_for_test(epoch).await.unwrap();
+                storage_2.init_for_test(epoch).await.unwrap();
             }
 
             let (storage, other) = if index % 2 == 0 {
@@ -731,8 +767,13 @@ pub(crate) mod tests {
 
         // 4. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let (mut result_task, task_stats) =
-            compact(Arc::new(compact_ctx), compact_task.clone(), rx).await;
+        let (mut result_task, task_stats) = compact(
+            compact_ctx,
+            compact_task.clone(),
+            rx,
+            Box::new(sstable_object_id_manager.clone()),
+        )
+        .await;
 
         hummock_manager_ref
             .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
@@ -815,13 +856,27 @@ pub(crate) mod tests {
             TableId::from(existing_table_id),
         )
         .await;
-        let filter_key_extractor_manager = storage.filter_key_extractor_manager().clone();
+
+        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
+        {
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                rpc_filter_key_extractor_manager,
+            ) => rpc_filter_key_extractor_manager,
+            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+        };
+
         let compact_ctx = get_compactor_context_with_filter_key_extractor_manager(
             &storage,
-            &hummock_meta_client,
-            filter_key_extractor_manager.clone(),
+            rpc_filter_key_extractor_manager.clone(),
         );
-        filter_key_extractor_manager.update(
+        let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+            hummock_meta_client.clone(),
+            storage
+                .storage_opts()
+                .clone()
+                .sstable_id_remote_fetch_number,
+        ));
+        rpc_filter_key_extractor_manager.update(
             2,
             Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
         );
@@ -843,7 +898,7 @@ pub(crate) mod tests {
             epoch += millisec_interval_epoch;
             let next_epoch = epoch + millisec_interval_epoch;
             if i == 0 {
-                local.init(epoch);
+                local.init_for_test(epoch).await.unwrap();
             }
             epoch_set.insert(epoch);
             let mut prefix = BytesMut::default();
@@ -900,8 +955,13 @@ pub(crate) mod tests {
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let (mut result_task, task_stats) =
-            compact(Arc::new(compact_ctx), compact_task.clone(), rx).await;
+        let (mut result_task, task_stats) = compact(
+            compact_ctx,
+            compact_task.clone(),
+            rx,
+            Box::new(sstable_object_id_manager.clone()),
+        )
+        .await;
 
         hummock_manager_ref
             .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
@@ -990,20 +1050,31 @@ pub(crate) mod tests {
         )
         .await;
 
-        let filter_key_extractor_manager = storage.filter_key_extractor_manager().clone();
-        filter_key_extractor_manager.update(
+        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
+        {
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                rpc_filter_key_extractor_manager,
+            ) => rpc_filter_key_extractor_manager,
+            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+        };
+
+        rpc_filter_key_extractor_manager.update(
             existing_table_id,
             Arc::new(FilterKeyExtractorImpl::FixedLength(
                 FixedLengthFilterKeyExtractor::new(TABLE_PREFIX_LEN + key_prefix.len()),
             )),
         );
-
         let compact_ctx = get_compactor_context_with_filter_key_extractor_manager(
             &storage,
-            &hummock_meta_client,
-            filter_key_extractor_manager.clone(),
+            rpc_filter_key_extractor_manager,
         );
-
+        let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+            hummock_meta_client.clone(),
+            storage
+                .storage_opts()
+                .clone()
+                .sstable_id_remote_fetch_number,
+        ));
         // 1. add sstables
         let val = Bytes::from(b"0"[..].to_vec()); // 1 Byte value
         let kv_count = 11;
@@ -1019,7 +1090,7 @@ pub(crate) mod tests {
         for i in 0..kv_count {
             epoch += millisec_interval_epoch;
             if i == 0 {
-                local.init(epoch);
+                local.init_for_test(epoch).await.unwrap();
             }
             let next_epoch = epoch + millisec_interval_epoch;
             epoch_set.insert(epoch);
@@ -1069,8 +1140,13 @@ pub(crate) mod tests {
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let (mut result_task, task_stats) =
-            compact(Arc::new(compact_ctx), compact_task.clone(), rx).await;
+        let (mut result_task, task_stats) = compact(
+            compact_ctx,
+            compact_task.clone(),
+            rx,
+            Box::new(sstable_object_id_manager.clone()),
+        )
+        .await;
 
         hummock_manager_ref
             .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
@@ -1166,14 +1242,19 @@ pub(crate) mod tests {
             TableId::from(existing_table_id),
         )
         .await;
-        let compact_ctx =
-            prepare_compactor_and_filter(&storage, &hummock_meta_client, existing_table_id);
-
+        let compact_ctx = prepare_compactor_and_filter(&storage, existing_table_id);
+        let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+            hummock_meta_client.clone(),
+            storage
+                .storage_opts()
+                .clone()
+                .sstable_id_remote_fetch_number,
+        ));
         prepare_data(hummock_meta_client.clone(), &storage, existing_table_id, 2).await;
         let mut local = storage
             .new_local(NewLocalOptions::for_test(existing_table_id.into()))
             .await;
-        local.init(130);
+        local.init_for_test(130).await.unwrap();
         let prefix_key_range = |k: u16| {
             let key = k.to_be_bytes();
             (
@@ -1217,8 +1298,13 @@ pub(crate) mod tests {
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let (mut result_task, task_stats) =
-            compact(Arc::new(compact_ctx), compact_task.clone(), rx).await;
+        let (mut result_task, task_stats) = compact(
+            compact_ctx,
+            compact_task.clone(),
+            rx,
+            Box::new(sstable_object_id_manager.clone()),
+        )
+        .await;
 
         hummock_manager_ref
             .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
diff --git a/src/storage/hummock_test/src/failpoint_tests.rs b/src/storage/hummock_test/src/failpoint_tests.rs
index 25a6312a3ae38..83af92d469afe 100644
--- a/src/storage/hummock_test/src/failpoint_tests.rs
+++ b/src/storage/hummock_test/src/failpoint_tests.rs
@@ -33,6 +33,7 @@ use risingwave_storage::store::{
 use risingwave_storage::StateStore;
 
 use crate::get_notification_client_for_test;
+use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 use crate::test_utils::TestIngestBatch;
 
 #[tokio::test]
@@ -74,7 +75,7 @@ async fn test_failpoints_state_store_read_upload() {
     ];
     // Make sure the batch is sorted.
     batch2.sort_by(|(k1, _), (k2, _)| k1.cmp(k2));
-    local.init(1);
+    local.init_for_test(1).await.unwrap();
     local
         .ingest_batch(
             batch1,
diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs
index a18c654920635..4e3d6e1aed919 100644
--- a/src/storage/hummock_test/src/hummock_storage_tests.rs
+++ b/src/storage/hummock_test/src/hummock_storage_tests.rs
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::sync::Arc;
 
@@ -29,6 +28,7 @@ use risingwave_storage::storage_value::StorageValue;
 use risingwave_storage::store::*;
 use risingwave_storage::StateStore;
 
+use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 use crate::test_utils::{prepare_hummock_test_env, TestIngestBatch};
 
 #[tokio::test]
@@ -92,7 +92,7 @@ async fn test_storage_basic() {
 
     // epoch 0 is reserved by storage service
     let epoch1: u64 = 1;
-    hummock_storage.init(epoch1);
+    hummock_storage.init_for_test(epoch1).await.unwrap();
 
     // Write the first batch.
     hummock_storage
@@ -471,7 +471,7 @@ async fn test_state_store_sync() {
     let read_version = hummock_storage.read_version();
 
     let epoch1 = read_version.read().committed().max_committed_epoch() + 1;
-    hummock_storage.init(epoch1);
+    hummock_storage.init_for_test(epoch1).await.unwrap();
 
     // ingest 16B batch
     let mut batch1 = vec![
@@ -789,7 +789,7 @@ async fn test_delete_get() {
 
     let epoch1 = initial_epoch + 1;
 
-    hummock_storage.init(epoch1);
+    hummock_storage.init_for_test(epoch1).await.unwrap();
     let batch1 = vec![
         (Bytes::from("aa"), StorageValue::new_put("111")),
         (Bytes::from("bb"), StorageValue::new_put("222")),
@@ -866,7 +866,7 @@ async fn test_multiple_epoch_sync() {
         .max_committed_epoch();
 
     let epoch1 = initial_epoch + 1;
-    hummock_storage.init(epoch1);
+    hummock_storage.init_for_test(epoch1).await.unwrap();
     let batch1 = vec![
         (Bytes::from("aa"), StorageValue::new_put("111")),
         (Bytes::from("bb"), StorageValue::new_put("222")),
@@ -1014,7 +1014,7 @@ async fn test_iter_with_min_epoch() {
         })
         .collect();
 
-    hummock_storage.init(epoch1);
+    hummock_storage.init_for_test(epoch1).await.unwrap();
 
     hummock_storage
         .ingest_batch(
@@ -1255,7 +1255,7 @@ async fn test_hummock_version_reader() {
         })
         .collect();
     {
-        hummock_storage.init(epoch1);
+        hummock_storage.init_for_test(epoch1).await.unwrap();
         hummock_storage
             .ingest_batch(
                 batch_epoch1,
@@ -1657,7 +1657,7 @@ async fn test_get_with_min_epoch() {
         .await;
 
     let epoch1 = (31 * 1000) << 16;
-    hummock_storage.init(epoch1);
+    hummock_storage.init_for_test(epoch1).await.unwrap();
 
     let gen_key = |index: usize| -> Vec<u8> {
         UserKey::for_test(TEST_TABLE_ID, format!("key_{}", index)).encode()
diff --git a/src/storage/hummock_test/src/lib.rs b/src/storage/hummock_test/src/lib.rs
index 8826c58c21fe4..73e1d8cd0eaad 100644
--- a/src/storage/hummock_test/src/lib.rs
+++ b/src/storage/hummock_test/src/lib.rs
@@ -17,6 +17,7 @@
 #![feature(bound_map)]
 #![feature(type_alias_impl_trait)]
 #![feature(associated_type_bounds)]
+#![feature(return_position_impl_trait_in_trait)]
 
 #[cfg(test)]
 mod compactor_tests;
@@ -39,4 +40,9 @@ mod hummock_storage_tests;
 mod mock_notification_client;
 #[cfg(all(test, feature = "sync_point"))]
 mod sync_point_tests;
+
+// Not feature gated by #[cfg(test)] because it is used by test binaries e.g. compaction_test,
+// not just tests.
+pub mod local_state_store_test_utils;
+
 pub use mock_notification_client::get_notification_client_for_test;
diff --git a/src/storage/hummock_test/src/local_state_store_test_utils.rs b/src/storage/hummock_test/src/local_state_store_test_utils.rs
new file mode 100644
index 0000000000000..d099b8883589e
--- /dev/null
+++ b/src/storage/hummock_test/src/local_state_store_test_utils.rs
@@ -0,0 +1,28 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+
+use risingwave_common::util::epoch::EpochPair;
+use risingwave_storage::error::StorageResult;
+use risingwave_storage::store::{InitOptions, LocalStateStore};
+
+pub trait LocalStateStoreTestExt: LocalStateStore {
+    fn init_for_test(&mut self, epoch: u64) -> impl Future<Output = StorageResult<()>> + Send + '_ {
+        self.init(InitOptions::new_with_epoch(EpochPair::new_test_epoch(
+            epoch,
+        )))
+    }
+}
+impl<T: LocalStateStore> LocalStateStoreTestExt for T {}
diff --git a/src/storage/hummock_test/src/local_version_manager_tests.rs b/src/storage/hummock_test/src/local_version_manager_tests.rs
index cff1213bc97fc..b07b8882bdf3e 100644
--- a/src/storage/hummock_test/src/local_version_manager_tests.rs
+++ b/src/storage/hummock_test/src/local_version_manager_tests.rs
@@ -45,8 +45,8 @@ use crate::test_utils::prepare_first_valid_version;
 
 pub async fn prepare_local_version_manager(
     opt: Arc<StorageOpts>,
-    env: MetaSrvEnv<MemStore>,
-    hummock_manager_ref: HummockManagerRef<MemStore>,
+    env: MetaSrvEnv,
+    hummock_manager_ref: HummockManagerRef,
     worker_node: WorkerNode,
 ) -> LocalVersionManagerRef {
     let (pinned_version, _, _) =
diff --git a/src/storage/hummock_test/src/mock_notification_client.rs b/src/storage/hummock_test/src/mock_notification_client.rs
index 183bb06014609..b88f0e467c9b1 100644
--- a/src/storage/hummock_test/src/mock_notification_client.rs
+++ b/src/storage/hummock_test/src/mock_notification_client.rs
@@ -20,24 +20,23 @@ use risingwave_common::util::addr::HostAddr;
 use risingwave_common_service::observer_manager::{Channel, NotificationClient};
 use risingwave_meta::hummock::{HummockManager, HummockManagerRef};
 use risingwave_meta::manager::{MessageStatus, MetaSrvEnv, NotificationManagerRef, WorkerKey};
-use risingwave_meta::storage::{MemStore, MetaStore};
 use risingwave_pb::backup_service::MetaBackupManifestId;
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::hummock::WriteLimits;
 use risingwave_pb::meta::{MetaSnapshot, SubscribeResponse, SubscribeType};
 use tokio::sync::mpsc::UnboundedReceiver;
 
-pub struct MockNotificationClient<S: MetaStore> {
+pub struct MockNotificationClient {
     addr: HostAddr,
-    notification_manager: NotificationManagerRef<S>,
-    hummock_manager: HummockManagerRef<S>,
+    notification_manager: NotificationManagerRef,
+    hummock_manager: HummockManagerRef,
 }
 
-impl<S: MetaStore> MockNotificationClient<S> {
+impl MockNotificationClient {
     pub fn new(
         addr: HostAddr,
-        notification_manager: NotificationManagerRef<S>,
-        hummock_manager: HummockManagerRef<S>,
+        notification_manager: NotificationManagerRef,
+        hummock_manager: HummockManagerRef,
     ) -> Self {
         Self {
             addr,
@@ -48,7 +47,7 @@ impl<S: MetaStore> MockNotificationClient<S> {
 }
 
 #[async_trait::async_trait]
-impl<S: MetaStore> NotificationClient for MockNotificationClient<S> {
+impl NotificationClient for MockNotificationClient {
     type Channel = TestChannel<SubscribeResponse>;
 
     async fn subscribe(&self, subscribe_type: SubscribeType) -> Result<Self::Channel> {
@@ -78,10 +77,10 @@ impl<S: MetaStore> NotificationClient for MockNotificationClient<S> {
 }
 
 pub fn get_notification_client_for_test(
-    env: MetaSrvEnv<MemStore>,
-    hummock_manager_ref: Arc<HummockManager<MemStore>>,
+    env: MetaSrvEnv,
+    hummock_manager_ref: Arc<HummockManager>,
     worker_node: WorkerNode,
-) -> MockNotificationClient<MemStore> {
+) -> MockNotificationClient {
     MockNotificationClient::new(
         worker_node.get_host().unwrap().into(),
         env.notification_manager_ref(),
diff --git a/src/storage/hummock_test/src/snapshot_tests.rs b/src/storage/hummock_test/src/snapshot_tests.rs
index a345ec3e0f1ea..3870dae070903 100644
--- a/src/storage/hummock_test/src/snapshot_tests.rs
+++ b/src/storage/hummock_test/src/snapshot_tests.rs
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 use std::ops::Bound;
 use std::sync::Arc;
 
@@ -27,6 +26,7 @@ use risingwave_storage::store::{
     LocalStateStore, NewLocalOptions, PrefetchOptions, ReadOptions, WriteOptions,
 };
 
+use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 use crate::test_utils::{with_hummock_storage_v2, HummockStateStoreTestTrait, TestIngestBatch};
 
 macro_rules! assert_count_range_scan {
@@ -42,13 +42,9 @@ macro_rules! assert_count_range_scan {
                 bounds,
                 $epoch,
                 ReadOptions {
-                    ignore_range_tombstone: false,
-                    prefix_hint: None,
-                    table_id: Default::default(),
-                    retention_seconds: None,
-                    read_version_from_backup: false,
                     prefetch_options: PrefetchOptions::new_for_exhaust_iter(),
                     cache_policy: CachePolicy::Fill(CachePriority::High),
+                    ..Default::default()
                 },
             )
             .await
@@ -110,7 +106,7 @@ async fn test_snapshot_inner(
         .await;
 
     let epoch1: u64 = 1;
-    local.init(epoch1);
+    local.init_for_test(epoch1).await.unwrap();
     local
         .ingest_batch(
             vec![
@@ -231,7 +227,7 @@ async fn test_snapshot_range_scan_inner(
     let mut local = hummock_storage
         .new_local(NewLocalOptions::for_test(Default::default()))
         .await;
-    local.init(epoch);
+    local.init_for_test(epoch).await.unwrap();
 
     local
         .ingest_batch(
diff --git a/src/storage/hummock_test/src/state_store_tests.rs b/src/storage/hummock_test/src/state_store_tests.rs
index 1fa74ad2eabcf..bc68da1f9d298 100644
--- a/src/storage/hummock_test/src/state_store_tests.rs
+++ b/src/storage/hummock_test/src/state_store_tests.rs
@@ -37,6 +37,7 @@ use risingwave_storage::storage_value::StorageValue;
 use risingwave_storage::store::*;
 
 use crate::get_notification_client_for_test;
+use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 use crate::test_utils::{with_hummock_storage_v2, HummockStateStoreTestTrait, TestIngestBatch};
 
 #[tokio::test]
@@ -115,7 +116,7 @@ async fn test_basic_inner(
 
     // epoch 0 is reserved by storage service
     let epoch1: u64 = 1;
-    local.init(epoch1);
+    local.init_for_test(epoch1).await.unwrap();
 
     // try to write an empty batch, and hummock should write nothing
     let size = local
@@ -401,7 +402,7 @@ async fn test_state_store_sync_inner(
     let mut local = hummock_storage
         .new_local(NewLocalOptions::for_test(Default::default()))
         .await;
-    local.init(epoch);
+    local.init_for_test(epoch).await.unwrap();
     local
         .ingest_batch(
             batch1,
@@ -814,7 +815,7 @@ async fn test_write_anytime_inner(
     ];
 
     let mut local = hummock_storage.new_local(NewLocalOptions::default()).await;
-    local.init(epoch1);
+    local.init_for_test(epoch1).await.unwrap();
 
     local
         .ingest_batch(
@@ -1009,7 +1010,7 @@ async fn test_delete_get_inner(
         (Bytes::from("bb"), StorageValue::new_put("222")),
     ];
     let mut local = hummock_storage.new_local(NewLocalOptions::default()).await;
-    local.init(epoch1);
+    local.init_for_test(epoch1).await.unwrap();
     local
         .ingest_batch(
             batch1,
@@ -1085,7 +1086,7 @@ async fn test_multiple_epoch_sync_inner(
     ];
 
     let mut local = hummock_storage.new_local(NewLocalOptions::default()).await;
-    local.init(epoch1);
+    local.init_for_test(epoch1).await.unwrap();
     local
         .ingest_batch(
             batch1,
@@ -1217,7 +1218,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() {
 
     let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch();
     let epoch1 = initial_epoch + 1;
-    local_hummock_storage.init(epoch1);
+    local_hummock_storage.init_for_test(epoch1).await.unwrap();
     local_hummock_storage
         .insert(Bytes::from("aa"), Bytes::from("111"), None)
         .unwrap();
@@ -1338,7 +1339,7 @@ async fn test_replicated_local_hummock_storage() {
 
     let epoch1 = epoch0 + 1;
 
-    local_hummock_storage.init(epoch1);
+    local_hummock_storage.init_for_test(epoch1).await.unwrap();
     // ingest 16B batch
     let mut batch1 = vec![
         (
@@ -1408,7 +1409,7 @@ async fn test_replicated_local_hummock_storage() {
         .new_local(NewLocalOptions::for_test(TEST_TABLE_ID))
         .await;
 
-    local_hummock_storage_2.init(epoch2);
+    local_hummock_storage_2.init_for_test(epoch2).await.unwrap();
 
     // ingest 16B batch
     let mut batch2 = vec![
diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs
index 9533119ec3733..7194dd2d963ea 100644
--- a/src/storage/hummock_test/src/sync_point_tests.rs
+++ b/src/storage/hummock_test/src/sync_point_tests.rs
@@ -34,12 +34,11 @@ use risingwave_meta::hummock::test_utils::{
 };
 use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
 use risingwave_meta::manager::LocalNotification;
-use risingwave_meta::storage::MemStore;
 use risingwave_pb::hummock::compact_task::TaskStatus;
 use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::hummock::compactor::compactor_runner::compact;
 use risingwave_storage::hummock::compactor::CompactorContext;
-use risingwave_storage::hummock::{CachePolicy, SstableObjectIdManager};
+use risingwave_storage::hummock::{CachePolicy, GetObjectId, SstableObjectIdManager};
 use risingwave_storage::store::{LocalStateStore, NewLocalOptions, ReadOptions};
 use risingwave_storage::StateStore;
 use serial_test::serial;
@@ -48,6 +47,7 @@ use super::compactor_tests::tests::{
     flush_and_commit, get_hummock_storage, prepare_compactor_and_filter,
 };
 use crate::get_notification_client_for_test;
+use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 
 #[tokio::test]
 #[cfg(feature = "sync_point")]
@@ -73,7 +73,7 @@ async fn test_syncpoints_sstable_object_id_manager() {
     });
 
     // Start the task that fetches new ids.
-    let sstable_object_id_manager_clone = sstable_object_id_manager.clone();
+    let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone();
     let leader_task = tokio::spawn(async move {
         sstable_object_id_manager_clone
             .get_new_sst_object_id()
@@ -90,7 +90,7 @@ async fn test_syncpoints_sstable_object_id_manager() {
     // Start tasks that waits to be notified.
     let mut follower_tasks = vec![];
     for _ in 0..3 {
-        let sstable_object_id_manager_clone = sstable_object_id_manager.clone();
+        let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone();
         let follower_task = tokio::spawn(async move {
             sstable_object_id_manager_clone
                 .get_new_sst_object_id()
@@ -137,7 +137,7 @@ async fn test_syncpoints_test_failpoints_fetch_ids() {
     });
 
     // Start the task that fetches new ids.
-    let sstable_object_id_manager_clone = sstable_object_id_manager.clone();
+    let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone();
     let leader_task = tokio::spawn(async move {
         fail::cfg("get_new_sst_ids_err", "return").unwrap();
         sstable_object_id_manager_clone
@@ -153,7 +153,7 @@ async fn test_syncpoints_test_failpoints_fetch_ids() {
     // Start tasks that waits to be notified.
     let mut follower_tasks = vec![];
     for _ in 0..3 {
-        let sstable_object_id_manager_clone = sstable_object_id_manager.clone();
+        let mut sstable_object_id_manager_clone = sstable_object_id_manager.clone();
         let follower_task = tokio::spawn(async move {
             sstable_object_id_manager_clone
                 .get_new_sst_object_id()
@@ -228,8 +228,9 @@ async fn test_syncpoints_test_local_notification_receiver() {
 }
 
 pub async fn compact_once(
-    hummock_manager_ref: HummockManagerRef<MemStore>,
-    compact_ctx: Arc<CompactorContext>,
+    hummock_manager_ref: HummockManagerRef,
+    compact_ctx: CompactorContext,
+    sstable_object_id_manager: Arc<SstableObjectIdManager>,
 ) {
     // 2. get compact task
     let manual_compcation_option = ManualCompactionOption {
@@ -251,7 +252,13 @@ pub async fn compact_once(
     compact_task.compaction_filter_mask = compaction_filter_flag.bits();
     // 3. compact
     let (_tx, rx) = tokio::sync::oneshot::channel();
-    let (mut result_task, task_stats) = compact(compact_ctx, compact_task.clone(), rx).await;
+    let (mut result_task, task_stats) = compact(
+        compact_ctx,
+        compact_task.clone(),
+        rx,
+        Box::new(sstable_object_id_manager),
+    )
+    .await;
 
     hummock_manager_ref
         .report_compact_task(&mut result_task, Some(to_prost_table_stats_map(task_stats)))
@@ -282,10 +289,14 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
         TableId::from(existing_table_id),
     )
     .await;
-    let compact_ctx = Arc::new(prepare_compactor_and_filter(
-        &storage,
-        &hummock_meta_client,
-        existing_table_id,
+    let compact_ctx = prepare_compactor_and_filter(&storage, existing_table_id);
+
+    let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
+        hummock_meta_client.clone(),
+        storage
+            .storage_opts()
+            .clone()
+            .sstable_id_remote_fetch_number,
     ));
 
     let mut local = storage
@@ -296,7 +307,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     let val0 = Bytes::from(b"0"[..].repeat(1 << 10)); // 1024 Byte value
     let val1 = Bytes::from(b"1"[..].repeat(1 << 10)); // 1024 Byte value
 
-    local.init(100);
+    local.init_for_test(100).await.unwrap();
     let mut start_key = b"\0\0aaa".to_vec();
     for _ in 0..10 {
         local
@@ -320,7 +331,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     local.flush(Vec::new()).await.unwrap();
     local.seal_current_epoch(101);
     flush_and_commit(&hummock_meta_client, &storage, 100).await;
-    compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await;
+    compact_once(
+        hummock_manager_ref.clone(),
+        compact_ctx.clone(),
+        sstable_object_id_manager.clone(),
+    )
+    .await;
 
     local
         .insert(Bytes::from(b"\0\0aaa".as_slice()), val1.clone(), None)
@@ -337,7 +353,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
         .unwrap();
     local.seal_current_epoch(102);
     flush_and_commit(&hummock_meta_client, &storage, 101).await;
-    compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await;
+    compact_once(
+        hummock_manager_ref.clone(),
+        compact_ctx.clone(),
+        sstable_object_id_manager.clone(),
+    )
+    .await;
 
     local
         .insert(Bytes::from(b"\0\0hhh".as_slice()), val1.clone(), None)
@@ -355,7 +376,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     local.seal_current_epoch(103);
     flush_and_commit(&hummock_meta_client, &storage, 102).await;
     // move this two file to the same level.
-    compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await;
+    compact_once(
+        hummock_manager_ref.clone(),
+        compact_ctx.clone(),
+        sstable_object_id_manager.clone(),
+    )
+    .await;
 
     local
         .insert(Bytes::from(b"\0\0lll".as_slice()), val1.clone(), None)
@@ -367,7 +393,12 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     local.seal_current_epoch(u64::MAX);
     flush_and_commit(&hummock_meta_client, &storage, 103).await;
     // move this two file to the same level.
-    compact_once(hummock_manager_ref.clone(), compact_ctx.clone()).await;
+    compact_once(
+        hummock_manager_ref.clone(),
+        compact_ctx.clone(),
+        sstable_object_id_manager.clone(),
+    )
+    .await;
 
     // 4. get the latest version and check
     let version = hummock_manager_ref.get_current_version().await;
diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs
index 523637f138a72..d69c835930aa2 100644
--- a/src/storage/hummock_test/src/test_utils.rs
+++ b/src/storage/hummock_test/src/test_utils.rs
@@ -25,15 +25,14 @@ use risingwave_meta::hummock::test_utils::{
 };
 use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
 use risingwave_meta::manager::MetaSrvEnv;
-use risingwave_meta::storage::{MemStore, MetaStore};
 use risingwave_pb::catalog::{PbTable, Table};
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::hummock::version_update_payload;
 use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::error::StorageResult;
 use risingwave_storage::filter_key_extractor::{
-    FilterKeyExtractorImpl, FilterKeyExtractorManager, FilterKeyExtractorManagerRef,
-    FullKeyFilterKeyExtractor,
+    FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor,
+    RpcFilterKeyExtractorManager,
 };
 use risingwave_storage::hummock::backup_reader::BackupReader;
 use risingwave_storage::hummock::event_handler::HummockEvent;
@@ -50,8 +49,8 @@ use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
 use crate::mock_notification_client::get_notification_client_for_test;
 
 pub async fn prepare_first_valid_version(
-    env: MetaSrvEnv<MemStore>,
-    hummock_manager_ref: HummockManagerRef<MemStore>,
+    env: MetaSrvEnv,
+    hummock_manager_ref: HummockManagerRef,
     worker_node: WorkerNode,
 ) -> (
     PinnedVersion,
@@ -66,7 +65,7 @@ pub async fn prepare_first_valid_version(
     let observer_manager = ObserverManager::new(
         notification_client,
         HummockObserverNode::new(
-            Arc::new(FilterKeyExtractorManager::default()),
+            Arc::new(RpcFilterKeyExtractorManager::default()),
             backup_manager,
             tx.clone(),
             write_limiter,
@@ -163,20 +162,27 @@ pub async fn with_hummock_storage_v2(
     (hummock_storage, meta_client)
 }
 pub fn update_filter_key_extractor_for_table_ids(
-    filter_key_extractor_manager_ref: &FilterKeyExtractorManagerRef,
+    filter_key_extractor_manager_ref: &FilterKeyExtractorManager,
     table_ids: &[u32],
 ) {
+    let rpc_filter_key_extractor_manager = match filter_key_extractor_manager_ref {
+        FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+            rpc_filter_key_extractor_manager,
+        ) => rpc_filter_key_extractor_manager,
+        FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+    };
+
     for table_id in table_ids {
-        filter_key_extractor_manager_ref.update(
+        rpc_filter_key_extractor_manager.update(
             *table_id,
             Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
         )
     }
 }
 
-pub async fn register_tables_with_id_for_test<S: MetaStore>(
-    filter_key_extractor_manager: &FilterKeyExtractorManagerRef,
-    hummock_manager_ref: &HummockManagerRef<S>,
+pub async fn register_tables_with_id_for_test(
+    filter_key_extractor_manager: &FilterKeyExtractorManager,
+    hummock_manager_ref: &HummockManagerRef,
     table_ids: &[u32],
 ) {
     update_filter_key_extractor_for_table_ids(filter_key_extractor_manager, table_ids);
@@ -189,19 +195,25 @@ pub async fn register_tables_with_id_for_test<S: MetaStore>(
 }
 
 pub fn update_filter_key_extractor_for_tables(
-    filter_key_extractor_manager_ref: &FilterKeyExtractorManagerRef,
+    filter_key_extractor_manager: &FilterKeyExtractorManager,
     tables: &[PbTable],
 ) {
+    let rpc_filter_key_extractor_manager = match filter_key_extractor_manager {
+        FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+            rpc_filter_key_extractor_manager,
+        ) => rpc_filter_key_extractor_manager,
+        FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+    };
     for table in tables {
-        filter_key_extractor_manager_ref.update(
+        rpc_filter_key_extractor_manager.update(
             table.id,
             Arc::new(FilterKeyExtractorImpl::from_table(table)),
         )
     }
 }
-pub async fn register_tables_with_catalog_for_test<S: MetaStore>(
-    filter_key_extractor_manager: &FilterKeyExtractorManagerRef,
-    hummock_manager_ref: &HummockManagerRef<S>,
+pub async fn register_tables_with_catalog_for_test(
+    filter_key_extractor_manager: &FilterKeyExtractorManager,
+    hummock_manager_ref: &HummockManagerRef,
     tables: &[Table],
 ) {
     update_filter_key_extractor_for_tables(filter_key_extractor_manager, tables);
@@ -216,7 +228,7 @@ pub async fn register_tables_with_catalog_for_test<S: MetaStore>(
 
 pub struct HummockTestEnv {
     pub storage: HummockStorage,
-    pub manager: HummockManagerRef<MemStore>,
+    pub manager: HummockManagerRef,
     pub meta_client: Arc<MockHummockMetaClient>,
 }
 
diff --git a/src/storage/hummock_trace/src/collector.rs b/src/storage/hummock_trace/src/collector.rs
index 1db7fc16d2003..c6bd3a7093f80 100644
--- a/src/storage/hummock_trace/src/collector.rs
+++ b/src/storage/hummock_trace/src/collector.rs
@@ -34,7 +34,7 @@ use tokio::task_local;
 use crate::write::{TraceWriter, TraceWriterImpl};
 use crate::{
     ConcurrentIdGenerator, Operation, OperationResult, Record, RecordId, RecordIdGenerator,
-    TracedNewLocalOptions, TracedReadOptions, TracedSubResp, UniqueIdGenerator,
+    TracedInitOptions, TracedNewLocalOptions, TracedReadOptions, TracedSubResp, UniqueIdGenerator,
 };
 
 // Global collector instance used for trace collection
@@ -323,8 +323,11 @@ impl TraceSpan {
         )
     }
 
-    pub fn new_local_storage_init_span(epoch: u64, storage_type: StorageType) -> MayTraceSpan {
-        Self::new_global_op(Operation::LocalStorageInit(epoch), storage_type)
+    pub fn new_local_storage_init_span(
+        options: TracedInitOptions,
+        storage_type: StorageType,
+    ) -> MayTraceSpan {
+        Self::new_global_op(Operation::LocalStorageInit(options), storage_type)
     }
 
     pub fn send(&self, op: Operation) {
diff --git a/src/storage/hummock_trace/src/opts.rs b/src/storage/hummock_trace/src/opts.rs
index 89013d4dac020..8d9fa20af6ebf 100644
--- a/src/storage/hummock_trace/src/opts.rs
+++ b/src/storage/hummock_trace/src/opts.rs
@@ -15,6 +15,7 @@
 use bincode::{Decode, Encode};
 use risingwave_common::cache::CachePriority;
 use risingwave_common::catalog::{TableId, TableOption};
+use risingwave_common::util::epoch::EpochPair;
 use risingwave_hummock_sdk::HummockReadEpoch;
 
 use crate::TracedBytes;
@@ -183,3 +184,32 @@ impl From<TracedHummockReadEpoch> for HummockReadEpoch {
         }
     }
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Decode, Encode)]
+pub struct TracedEpochPair {
+    pub curr: TracedHummockEpoch,
+    pub prev: TracedHummockEpoch,
+}
+
+impl From<EpochPair> for TracedEpochPair {
+    fn from(value: EpochPair) -> Self {
+        TracedEpochPair {
+            curr: value.curr,
+            prev: value.prev,
+        }
+    }
+}
+
+impl From<TracedEpochPair> for EpochPair {
+    fn from(value: TracedEpochPair) -> Self {
+        EpochPair {
+            curr: value.curr,
+            prev: value.prev,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Decode, Encode)]
+pub struct TracedInitOptions {
+    pub epoch: TracedEpochPair,
+}
diff --git a/src/storage/hummock_trace/src/record.rs b/src/storage/hummock_trace/src/record.rs
index 8e8a6535f35a1..5d209635e967e 100644
--- a/src/storage/hummock_trace/src/record.rs
+++ b/src/storage/hummock_trace/src/record.rs
@@ -22,7 +22,8 @@ use prost::Message;
 use risingwave_pb::meta::SubscribeResponse;
 
 use crate::{
-    LocalStorageId, StorageType, TracedHummockReadEpoch, TracedNewLocalOptions, TracedReadOptions,
+    LocalStorageId, StorageType, TracedHummockReadEpoch, TracedInitOptions, TracedNewLocalOptions,
+    TracedReadOptions,
 };
 
 pub type RecordId = u64;
@@ -163,7 +164,7 @@ pub enum Operation {
     DropLocalStorage,
 
     /// Init of a local storage
-    LocalStorageInit(u64),
+    LocalStorageInit(TracedInitOptions),
 
     /// Try wait epoch
     TryWaitEpoch(TracedHummockReadEpoch),
diff --git a/src/storage/hummock_trace/src/replay/mod.rs b/src/storage/hummock_trace/src/replay/mod.rs
index 7fbadc140b9c1..41372f5fb2b5e 100644
--- a/src/storage/hummock_trace/src/replay/mod.rs
+++ b/src/storage/hummock_trace/src/replay/mod.rs
@@ -31,7 +31,10 @@ pub(crate) use worker::*;
 use crate::error::Result;
 #[cfg(test)]
 use crate::TraceError;
-use crate::{LocalStorageId, Record, TracedBytes, TracedNewLocalOptions, TracedReadOptions};
+use crate::{
+    LocalStorageId, Record, TracedBytes, TracedInitOptions, TracedNewLocalOptions,
+    TracedReadOptions,
+};
 
 pub type ReplayItem = (TracedBytes, TracedBytes);
 pub trait ReplayItemStream = Stream<Item = ReplayItem> + Send;
@@ -56,7 +59,7 @@ pub(crate) enum WorkerId {
 
 #[async_trait::async_trait]
 pub trait LocalReplay: LocalReplayRead + ReplayWrite + Send + Sync {
-    fn init(&mut self, epoch: u64);
+    async fn init(&mut self, options: TracedInitOptions) -> Result<()>;
     fn seal_current_epoch(&mut self, next_epoch: u64);
     fn is_dirty(&self) -> bool;
     fn epoch(&self) -> u64;
@@ -181,7 +184,7 @@ mock! {
     }
     #[async_trait::async_trait]
     impl LocalReplay for LocalReplayInterface{
-        fn init(&mut self, epoch: u64);
+        async fn init(&mut self, options: TracedInitOptions) -> Result<()>;
         fn seal_current_epoch(&mut self, next_epoch: u64);
         fn is_dirty(&self) -> bool;
         fn epoch(&self) -> u64;
diff --git a/src/storage/hummock_trace/src/replay/worker.rs b/src/storage/hummock_trace/src/replay/worker.rs
index f544c437ff20e..e5c2b168d6e0a 100644
--- a/src/storage/hummock_trace/src/replay/worker.rs
+++ b/src/storage/hummock_trace/src/replay/worker.rs
@@ -309,10 +309,10 @@ impl ReplayWorker {
                         .unwrap();
                 }
             }
-            Operation::LocalStorageInit(epoch) => {
+            Operation::LocalStorageInit(options) => {
                 assert_ne!(storage_type, StorageType::Global);
                 let local_storage = local_storages.get_mut(&storage_type).unwrap();
-                local_storage.init(epoch);
+                local_storage.init(options).await.unwrap();
             }
             Operation::TryWaitEpoch(epoch) => {
                 assert_eq!(storage_type, StorageType::Global);
diff --git a/src/storage/src/filter_key_extractor.rs b/src/storage/src/filter_key_extractor.rs
index 77dd68a198854..47ce552300e22 100644
--- a/src/storage/src/filter_key_extractor.rs
+++ b/src/storage/src/filter_key_extractor.rs
@@ -258,7 +258,6 @@ impl StateTableAccessor for FakeRemoteTableAccessor {
         )))
     }
 }
-
 struct FilterKeyExtractorManagerInner {
     table_id_to_filter_key_extractor: RwLock<HashMap<u32, Arc<FilterKeyExtractorImpl>>>,
     table_accessor: Box<dyn StateTableAccessor>,
@@ -334,19 +333,19 @@ impl FilterKeyExtractorManagerInner {
     }
 }
 
-/// `FilterKeyExtractorManager` is a wrapper for inner, and provide a protected read and write
+/// `RpcFilterKeyExtractorManager` is a wrapper for inner, and provide a protected read and write
 /// interface, its thread safe
-pub struct FilterKeyExtractorManager {
+pub struct RpcFilterKeyExtractorManager {
     inner: FilterKeyExtractorManagerInner,
 }
 
-impl Default for FilterKeyExtractorManager {
+impl Default for RpcFilterKeyExtractorManager {
     fn default() -> Self {
         Self::new(Box::<FakeRemoteTableAccessor>::default())
     }
 }
 
-impl FilterKeyExtractorManager {
+impl RpcFilterKeyExtractorManager {
     pub fn new(table_accessor: Box<dyn StateTableAccessor>) -> Self {
         Self {
             inner: FilterKeyExtractorManagerInner {
@@ -376,15 +375,60 @@ impl FilterKeyExtractorManager {
     /// Acquire a `MultiFilterKeyExtractor` by `table_id_set`
     /// Internally, try to get all `filter_key_extractor` from `hashmap`. Will block the caller if
     /// `table_id` does not util version update (notify), and retry to get
+    async fn acquire(&self, table_id_set: HashSet<u32>) -> HummockResult<FilterKeyExtractorImpl> {
+        self.inner.acquire(table_id_set).await
+    }
+}
+
+#[derive(Clone)]
+pub enum FilterKeyExtractorManager {
+    RpcFilterKeyExtractorManager(Arc<RpcFilterKeyExtractorManager>),
+    StaticFilterKeyExtractorManager(Arc<StaticFilterKeyExtractorManager>),
+}
+
+impl FilterKeyExtractorManager {
     pub async fn acquire(
         &self,
         table_id_set: HashSet<u32>,
     ) -> HummockResult<FilterKeyExtractorImpl> {
-        self.inner.acquire(table_id_set).await
+        match self {
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                rpc_filter_key_exactor_manager,
+            ) => rpc_filter_key_exactor_manager.acquire(table_id_set).await,
+            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(
+                static_filter_key_extractor_manager,
+            ) => static_filter_key_extractor_manager.acquire(table_id_set),
+        }
     }
 }
 
-pub type FilterKeyExtractorManagerRef = Arc<FilterKeyExtractorManager>;
+#[derive(Clone)]
+pub struct StaticFilterKeyExtractorManager {
+    id_to_table: HashMap<u32, Table>,
+}
+
+impl StaticFilterKeyExtractorManager {
+    pub fn new(id_to_table: HashMap<u32, Table>) -> Self {
+        Self { id_to_table }
+    }
+
+    fn acquire(&self, table_id_set: HashSet<u32>) -> HummockResult<FilterKeyExtractorImpl> {
+        let mut multi_filter_key_extractor = MultiFilterKeyExtractor::default();
+        for table_id in table_id_set {
+            if let Some(table) = self.id_to_table.get(&table_id) {
+                let key_extractor = Arc::new(FilterKeyExtractorImpl::from_table(table));
+                multi_filter_key_extractor.register(table_id, key_extractor);
+            } else {
+                return Err(HummockError::other(format!(
+                    "table {} is absent in id_to_table, need to request rpc list_tables to get the schema", table_id,
+                )));
+            }
+        }
+        Ok(FilterKeyExtractorImpl::Multi(multi_filter_key_extractor))
+    }
+}
+
+pub type FilterKeyExtractorManagerRef = Arc<RpcFilterKeyExtractorManager>;
 
 #[cfg(test)]
 mod tests {
@@ -410,10 +454,9 @@ mod tests {
 
     use super::{DummyFilterKeyExtractor, FilterKeyExtractor, SchemaFilterKeyExtractor};
     use crate::filter_key_extractor::{
-        FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor,
-        MultiFilterKeyExtractor,
+        FilterKeyExtractorImpl, FullKeyFilterKeyExtractor, MultiFilterKeyExtractor,
+        RpcFilterKeyExtractorManager,
     };
-
     const fn dummy_vnode() -> [u8; VirtualNode::SIZE] {
         VirtualNode::from_index(233).to_be_bytes()
     }
@@ -618,7 +661,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_filter_key_extractor_manager() {
-        let filter_key_extractor_manager = Arc::new(FilterKeyExtractorManager::default());
+        let filter_key_extractor_manager = Arc::new(RpcFilterKeyExtractorManager::default());
 
         filter_key_extractor_manager.update(
             1,
diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs
index 9a5a6c11e3eee..b1fd6b5411643 100644
--- a/src/storage/src/hummock/compactor/compaction_utils.rs
+++ b/src/storage/src/hummock/compactor/compaction_utils.rs
@@ -35,13 +35,13 @@ use crate::hummock::compactor::{
 use crate::hummock::multi_builder::TableBuilderFactory;
 use crate::hummock::sstable::DEFAULT_ENTRY_SIZE;
 use crate::hummock::{
-    CachePolicy, FilterBuilder, HummockResult, MemoryLimiter, SstableBuilder,
-    SstableBuilderOptions, SstableObjectIdManagerRef, SstableWriterFactory, SstableWriterOptions,
+    CachePolicy, FilterBuilder, GetObjectId, HummockResult, MemoryLimiter, SstableBuilder,
+    SstableBuilderOptions, SstableWriterFactory, SstableWriterOptions,
 };
 use crate::monitor::StoreLocalStatistic;
 
 pub struct RemoteBuilderFactory<W: SstableWriterFactory, F: FilterBuilder> {
-    pub sstable_object_id_manager: SstableObjectIdManagerRef,
+    pub object_id_getter: Box<dyn GetObjectId>,
     pub limiter: Arc<MemoryLimiter>,
     pub options: SstableBuilderOptions,
     pub policy: CachePolicy,
@@ -58,10 +58,7 @@ impl<W: SstableWriterFactory, F: FilterBuilder> TableBuilderFactory for RemoteBu
 
     async fn open_builder(&mut self) -> HummockResult<SstableBuilder<Self::Writer, Self::Filter>> {
         let timer = Instant::now();
-        let table_id = self
-            .sstable_object_id_manager
-            .get_new_sst_object_id()
-            .await?;
+        let table_id = self.object_id_getter.get_new_sst_object_id().await?;
         let cost = (timer.elapsed().as_secs_f64() * 1000000.0).round() as u64;
         self.remote_rpc_cost.fetch_add(cost, Ordering::Relaxed);
         let writer_options = SstableWriterOptions {
@@ -164,7 +161,7 @@ pub fn build_multi_compaction_filter(compact_task: &CompactTask) -> MultiCompact
 pub async fn generate_splits(
     sstable_infos: &Vec<SstableInfo>,
     compaction_size: u64,
-    context: Arc<CompactorContext>,
+    context: CompactorContext,
 ) -> HummockResult<Vec<KeyRange_vec>> {
     let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
     if compaction_size > parallel_compact_size {
@@ -234,7 +231,7 @@ pub async fn generate_splits(
     Ok(vec![])
 }
 
-pub fn estimate_task_output_capacity(context: Arc<CompactorContext>, task: &CompactTask) -> usize {
+pub fn estimate_task_output_capacity(context: CompactorContext, task: &CompactTask) -> usize {
     let max_target_file_size = context.storage_opts.sstable_size_mb as usize * (1 << 20);
     let total_input_uncompressed_file_size = task
         .input_ssts
diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs
index 3c4495515cff0..be13e32893d64 100644
--- a/src/storage/src/hummock/compactor/compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/compactor_runner.rs
@@ -45,7 +45,7 @@ use crate::hummock::sstable::CompactionDeleteRangesBuilder;
 use crate::hummock::value::HummockValue;
 use crate::hummock::{
     BlockedXor16FilterBuilder, CachePolicy, CompactionDeleteRanges, CompressionAlgorithm,
-    HummockResult, MonotonicDeleteEvent, SstableBuilderOptions, SstableStoreRef,
+    GetObjectId, HummockResult, MonotonicDeleteEvent, SstableBuilderOptions, SstableStoreRef,
 };
 use crate::monitor::{CompactorMetrics, StoreLocalStatistic};
 
@@ -58,7 +58,12 @@ pub struct CompactorRunner {
 }
 
 impl CompactorRunner {
-    pub fn new(split_index: usize, context: Arc<CompactorContext>, task: CompactTask) -> Self {
+    pub fn new(
+        split_index: usize,
+        context: CompactorContext,
+        task: CompactTask,
+        object_id_getter: Box<dyn GetObjectId>,
+    ) -> Self {
         let mut options: SstableBuilderOptions = context.storage_opts.as_ref().into();
         options.compression_algorithm = match task.compression_algorithm {
             0 => CompressionAlgorithm::None,
@@ -98,12 +103,13 @@ impl CompactorRunner {
                 split_weight_by_vnode: task.split_weight_by_vnode,
                 use_block_based_filter,
             },
+            object_id_getter,
         );
 
         Self {
             compactor,
             compact_task: task,
-            sstable_store: context.sstable_store.clone(),
+            sstable_store: context.sstable_store,
             key_range,
             split_index,
         }
@@ -232,35 +238,12 @@ impl CompactorRunner {
 /// Handles a compaction task and reports its status to hummock manager.
 /// Always return `Ok` and let hummock manager handle errors.
 pub async fn compact(
-    compactor_context: Arc<CompactorContext>,
+    compactor_context: CompactorContext,
     mut compact_task: CompactTask,
     mut shutdown_rx: Receiver<()>,
+    object_id_getter: Box<dyn GetObjectId>,
 ) -> (CompactTask, HashMap<u32, TableStats>) {
     let context = compactor_context.clone();
-    // Set a watermark SST id to prevent full GC from accidentally deleting SSTs for in-progress
-    // write op. The watermark is invalidated when this method exits.
-    let tracker_id = match context
-        .sstable_object_id_manager
-        .add_watermark_object_id(None)
-        .await
-    {
-        Ok(tracker_id) => tracker_id,
-        Err(err) => {
-            tracing::warn!("Failed to track pending SST object id. {:#?}", err);
-
-            // return TaskStatus::TrackSstObjectIdFailed;
-            compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed);
-            return (compact_task, HashMap::default());
-        }
-    };
-    let sstable_object_id_manager_clone = context.sstable_object_id_manager.clone();
-    let _guard = scopeguard::guard(
-        (tracker_id, sstable_object_id_manager_clone),
-        |(tracker_id, sstable_object_id_manager)| {
-            sstable_object_id_manager.remove_watermark_object_id(tracker_id);
-        },
-    );
-
     let group_label = compact_task.compaction_group_id.to_string();
     let cur_level_label = compact_task.input_ssts[0].level_idx.to_string();
     let select_table_infos = compact_task
@@ -330,7 +313,7 @@ pub async fn compact(
             .into_iter()
             .filter(|table_id| existing_table_ids.contains(table_id)),
     );
-    let multi_filter_key_extractor = match context
+    let multi_filter_key_extractor = match compactor_context
         .filter_key_extractor_manager
         .acquire(compact_table_ids.clone())
         .await
@@ -460,8 +443,12 @@ pub async fn compact(
     for (split_index, _) in compact_task.splits.iter().enumerate() {
         let filter = multi_filter.clone();
         let multi_filter_key_extractor = multi_filter_key_extractor.clone();
-        let compactor_runner =
-            CompactorRunner::new(split_index, compactor_context.clone(), compact_task.clone());
+        let compactor_runner = CompactorRunner::new(
+            split_index,
+            compactor_context.clone(),
+            compact_task.clone(),
+            object_id_getter.clone(),
+        );
         let del_agg = delete_range_agg.clone();
         let task_progress = task_progress_guard.progress.clone();
         let runner = async move {
@@ -559,7 +546,7 @@ pub async fn compact(
 /// Fills in the compact task and tries to report the task result to meta node.
 fn compact_done(
     mut compact_task: CompactTask,
-    context: Arc<CompactorContext>,
+    context: CompactorContext,
     output_ssts: Vec<CompactOutput>,
     task_status: TaskStatus,
 ) -> (CompactTask, HashMap<u32, TableStats>) {
diff --git a/src/storage/src/hummock/compactor/context.rs b/src/storage/src/hummock/compactor/context.rs
index 32a23c1fdbe98..ad3d5ffcc2dd6 100644
--- a/src/storage/src/hummock/compactor/context.rs
+++ b/src/storage/src/hummock/compactor/context.rs
@@ -16,13 +16,12 @@ use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 
 use parking_lot::RwLock;
-use risingwave_rpc_client::HummockMetaClient;
 
 use super::task_progress::TaskProgressManagerRef;
-use crate::filter_key_extractor::FilterKeyExtractorManagerRef;
+use crate::filter_key_extractor::FilterKeyExtractorManager;
 use crate::hummock::compactor::CompactionExecutor;
 use crate::hummock::sstable_store::SstableStoreRef;
-use crate::hummock::{MemoryLimiter, SstableObjectIdManagerRef};
+use crate::hummock::MemoryLimiter;
 use crate::monitor::CompactorMetrics;
 use crate::opts::StorageOpts;
 
@@ -32,9 +31,6 @@ pub struct CompactorContext {
     /// Storage options.
     pub storage_opts: Arc<StorageOpts>,
 
-    /// The meta client.
-    pub hummock_meta_client: Arc<dyn HummockMetaClient>,
-
     /// Sstable store that manages the sstables.
     pub sstable_store: SstableStoreRef,
 
@@ -46,12 +42,10 @@ pub struct CompactorContext {
 
     pub compaction_executor: Arc<CompactionExecutor>,
 
-    pub filter_key_extractor_manager: FilterKeyExtractorManagerRef,
+    pub filter_key_extractor_manager: FilterKeyExtractorManager,
 
     pub memory_limiter: Arc<MemoryLimiter>,
 
-    pub sstable_object_id_manager: SstableObjectIdManagerRef,
-
     pub task_progress_manager: TaskProgressManagerRef,
 
     pub await_tree_reg: Option<Arc<RwLock<await_tree::Registry<String>>>>,
@@ -63,10 +57,8 @@ impl CompactorContext {
     pub fn new_local_compact_context(
         storage_opts: Arc<StorageOpts>,
         sstable_store: SstableStoreRef,
-        hummock_meta_client: Arc<dyn HummockMetaClient>,
         compactor_metrics: Arc<CompactorMetrics>,
-        sstable_object_id_manager: SstableObjectIdManagerRef,
-        filter_key_extractor_manager: FilterKeyExtractorManagerRef,
+        filter_key_extractor_manager: FilterKeyExtractorManager,
     ) -> Self {
         let compaction_executor = if storage_opts.share_buffer_compaction_worker_threads_number == 0
         {
@@ -80,14 +72,12 @@ impl CompactorContext {
         // not limit memory for local compact
         Self {
             storage_opts,
-            hummock_meta_client,
             sstable_store,
             compactor_metrics,
             is_share_buffer_compact: true,
             compaction_executor,
             filter_key_extractor_manager,
             memory_limiter: MemoryLimiter::unlimit(),
-            sstable_object_id_manager,
             task_progress_manager: Default::default(),
             await_tree_reg: None,
             running_task_count: Arc::new(AtomicU32::new(0)),
diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs
index 1cc1270c472fc..c9d9d43c38785 100644
--- a/src/storage/src/hummock/compactor/mod.rs
+++ b/src/storage/src/hummock/compactor/mod.rs
@@ -23,7 +23,6 @@ pub(super) mod task_progress;
 
 use std::collections::HashMap;
 use std::marker::PhantomData;
-use std::ops::Div;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, SystemTime};
@@ -41,16 +40,16 @@ pub use iterator::{ConcatSstableIterator, SstableStreamIterator};
 use more_asserts::assert_ge;
 use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map;
 use risingwave_hummock_sdk::{HummockCompactionTaskId, LocalSstableInfo};
+use risingwave_pb::hummock::compact_task::TaskStatus;
 use risingwave_pb::hummock::subscribe_compaction_event_request::{
     Event as RequestEvent, HeartBeat, PullTask, ReportTask,
 };
 use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
 use risingwave_pb::hummock::{
-    CompactTaskProgress, CompactorWorkload, SubscribeCompactionEventRequest,
-    SubscribeCompactionEventResponse,
+    CompactTaskProgress, SubscribeCompactionEventRequest, SubscribeCompactionEventResponse,
 };
+use risingwave_rpc_client::HummockMetaClient;
 pub use shared_buffer_compact::{compact, merge_imms_in_memory};
-use sysinfo::{CpuRefreshKind, ProcessExt, ProcessRefreshKind, RefreshKind, System, SystemExt};
 use tokio::sync::oneshot::Sender;
 use tokio::task::JoinHandle;
 use tokio::time::Instant;
@@ -58,7 +57,10 @@ use tokio::time::Instant;
 pub use self::compaction_utils::{CompactionStatistics, RemoteBuilderFactory, TaskConfig};
 pub use self::task_progress::TaskProgress;
 use super::multi_builder::CapacitySplitTableBuilder;
-use super::{CompactionDeleteRanges, HummockResult, SstableBuilderOptions, Xor16FilterBuilder};
+use super::{
+    CompactionDeleteRanges, GetObjectId, HummockResult, SstableBuilderOptions,
+    SstableObjectIdManager, Xor16FilterBuilder,
+};
 use crate::filter_key_extractor::FilterKeyExtractorImpl;
 use crate::hummock::compactor::compactor_runner::compact_and_build_sst;
 use crate::hummock::iterator::{Forward, HummockIterator};
@@ -72,7 +74,8 @@ use crate::hummock::{
 /// Implementation of Hummock compaction.
 pub struct Compactor {
     /// The context of the compactor.
-    context: Arc<CompactorContext>,
+    context: CompactorContext,
+    object_id_getter: Box<dyn GetObjectId>,
     task_config: TaskConfig,
     options: SstableBuilderOptions,
     get_id_time: Arc<AtomicU64>,
@@ -83,15 +86,17 @@ pub type CompactOutput = (usize, Vec<LocalSstableInfo>, CompactionStatistics);
 impl Compactor {
     /// Create a new compactor.
     pub fn new(
-        context: Arc<CompactorContext>,
+        context: CompactorContext,
         options: SstableBuilderOptions,
         task_config: TaskConfig,
+        object_id_getter: Box<dyn GetObjectId>,
     ) -> Self {
         Self {
             context,
             options,
             task_config,
             get_id_time: Arc::new(AtomicU64::new(0)),
+            object_id_getter,
         }
     }
 
@@ -137,6 +142,7 @@ impl Compactor {
                     del_agg,
                     filter_key_extractor,
                     task_progress.clone(),
+                    self.object_id_getter.clone(),
                 )
                 .verbose_instrument_await("compact")
                 .await?
@@ -148,6 +154,7 @@ impl Compactor {
                     del_agg,
                     filter_key_extractor,
                     task_progress.clone(),
+                    self.object_id_getter.clone(),
                 )
                 .verbose_instrument_await("compact")
                 .await?
@@ -162,6 +169,7 @@ impl Compactor {
                     del_agg,
                     filter_key_extractor,
                     task_progress.clone(),
+                    self.object_id_getter.clone(),
                 )
                 .verbose_instrument_await("compact")
                 .await?
@@ -173,6 +181,7 @@ impl Compactor {
                     del_agg,
                     filter_key_extractor,
                     task_progress.clone(),
+                    self.object_id_getter.clone(),
                 )
                 .verbose_instrument_await("compact")
                 .await?
@@ -253,9 +262,10 @@ impl Compactor {
         del_agg: Arc<CompactionDeleteRanges>,
         filter_key_extractor: Arc<FilterKeyExtractorImpl>,
         task_progress: Option<Arc<TaskProgress>>,
+        object_id_getter: Box<dyn GetObjectId>,
     ) -> HummockResult<(Vec<SplitTableOutput>, CompactionStatistics)> {
         let builder_factory = RemoteBuilderFactory::<F, B> {
-            sstable_object_id_manager: self.context.sstable_object_id_manager.clone(),
+            object_id_getter,
             limiter: self.context.memory_limiter.clone(),
             options: self.options.clone(),
             policy: self.task_config.cache_policy,
@@ -297,17 +307,17 @@ impl Compactor {
 /// The background compaction thread that receives compaction tasks from hummock compaction
 /// manager and runs compaction tasks.
 #[cfg_attr(coverage, no_coverage)]
-pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<()>, Sender<()>) {
-    let hummock_meta_client = compactor_context.hummock_meta_client.clone();
+pub fn start_compactor(
+    compactor_context: CompactorContext,
+    hummock_meta_client: Arc<dyn HummockMetaClient>,
+    sstable_object_id_manager: Arc<SstableObjectIdManager>,
+) -> (JoinHandle<()>, Sender<()>) {
     type CompactionShutdownMap = Arc<Mutex<HashMap<u64, Sender<()>>>>;
     let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
     let stream_retry_interval = Duration::from_secs(30);
     let task_progress = compactor_context.task_progress_manager.clone();
     let periodic_event_update_interval = Duration::from_millis(1000);
     let cpu_core_num = compactor_context.compaction_executor.worker_num() as u32;
-    let mut system =
-        System::new_with_specifics(RefreshKind::new().with_cpu(CpuRefreshKind::everything()));
-    let pid = sysinfo::get_current_pid().unwrap();
     let running_task_count = compactor_context.running_task_count.clone();
     let pull_task_ack = Arc::new(AtomicBool::new(true));
 
@@ -323,7 +333,6 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
         let shutdown_map = CompactionShutdownMap::default();
         let mut min_interval = tokio::time::interval(stream_retry_interval);
         let mut periodic_event_interval = tokio::time::interval(periodic_event_update_interval);
-        let mut workload_collect_interval = tokio::time::interval(Duration::from_secs(60));
 
         // This outer loop is to recreate stream.
         'start_stream: loop {
@@ -358,7 +367,7 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
             pin_mut!(response_event_stream);
 
             let executor = compactor_context.compaction_executor.clone();
-            let mut last_workload = CompactorWorkload::default();
+            let sstable_object_id_manager = sstable_object_id_manager.clone();
 
             // This inner loop is to consume stream or report task progress.
             let mut event_loop_iteration_now = Instant::now();
@@ -437,7 +446,6 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
                         }
 
                         tracing::info!(
-                            cpu = %last_workload.cpu,
                             running_task_count = %running_task_count.load(Ordering::Relaxed),
                             pull_task_ack = %pull_task_ack.load(Ordering::Relaxed),
                             pending_pull_task_count = %pending_pull_task_count
@@ -446,26 +454,6 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
                         continue;
                     }
 
-                    _ = workload_collect_interval.tick() => {
-                        let refresh_result = system.refresh_process_specifics(pid, ProcessRefreshKind::new().with_cpu());
-                        debug_assert!(refresh_result);
-                        let cpu = if let Some(process) = system.process(pid) {
-                            process.cpu_usage().div(cpu_core_num as f32) as u32
-                        } else {
-                            tracing::warn!("fail to get process pid {:?}", pid);
-                            0
-                        };
-
-                        tracing::debug!("compactor cpu usage {cpu}");
-                        let workload = CompactorWorkload {
-                            cpu,
-                        };
-
-                        last_workload = workload.clone();
-
-                        continue;
-                    }
-
                     event = response_event_stream.next() => {
                         event
                     }
@@ -495,6 +483,7 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
                             .observe(consumed_latency_ms as _);
 
                         let meta_client = hummock_meta_client.clone();
+                        let sstable_object_id_manager = sstable_object_id_manager.clone();
                         executor.spawn(async move {
                                 let running_task_count = running_task_count.clone();
                                 match event {
@@ -503,7 +492,26 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
                                         let (tx, rx) = tokio::sync::oneshot::channel();
                                         let task_id = compact_task.task_id;
                                         shutdown.lock().unwrap().insert(task_id, tx);
-                                        let (compact_task, table_stats) = compactor_runner::compact(context, compact_task, rx).await;
+                                        let (compact_task, table_stats) = match sstable_object_id_manager.add_watermark_object_id(None).await
+                                        {
+                                            Ok(tracker_id) => {
+                                                let sstable_object_id_manager_clone = sstable_object_id_manager.clone();
+                                                let _guard = scopeguard::guard(
+                                                    (tracker_id, sstable_object_id_manager_clone),
+                                                    |(tracker_id, sstable_object_id_manager)| {
+                                                        sstable_object_id_manager.remove_watermark_object_id(tracker_id);
+                                                    },
+                                                );
+                                                compactor_runner::compact(context, compact_task, rx, Box::new(sstable_object_id_manager.clone())).await
+                                            },
+                                            Err(err) => {
+                                                tracing::warn!("Failed to track pending SST object id. {:#?}", err);
+                                                let mut compact_task = compact_task;
+                                                // return TaskStatus::TrackSstObjectIdFailed;
+                                                compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed);
+                                                (compact_task, HashMap::default())
+                                            }
+                                        };
                                         shutdown.lock().unwrap().remove(&task_id);
                                         running_task_count.fetch_sub(1, Ordering::SeqCst);
 
@@ -537,12 +545,14 @@ pub fn start_compactor(compactor_context: Arc<CompactorContext>) -> (JoinHandle<
                                         }
                                     }
                                     ResponseEvent::FullScanTask(full_scan_task) => {
-                                        Vacuum::full_scan(
-                                            full_scan_task,
-                                            context.sstable_store.clone(),
-                                            meta_client,
-                                        )
-                                        .await;
+                                        match Vacuum::handle_full_scan_task(full_scan_task, context.sstable_store.clone()).await {
+                                            Ok((object_ids, total_object_count, total_object_size)) => {
+                                                Vacuum::report_full_scan_task(object_ids, total_object_count, total_object_size, meta_client).await;
+                                            }
+                                            Err(e) => {
+                                                tracing::warn!("Failed to iter object: {:#?}", e);
+                                            }
+                                        }
                                     }
                                     ResponseEvent::ValidationTask(validation_task) => {
                                         validate_ssts(
diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
index 47b3475a77423..bcbaa19e2c3b1 100644
--- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs
+++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
@@ -46,7 +46,8 @@ use crate::hummock::utils::MemoryTracker;
 use crate::hummock::value::HummockValue;
 use crate::hummock::{
     create_monotonic_events_from_compaction_delete_events, BlockedXor16FilterBuilder, CachePolicy,
-    CompactionDeleteRanges, HummockError, HummockResult, SstableBuilderOptions,
+    CompactionDeleteRanges, GetObjectId, HummockError, HummockResult, SstableBuilderOptions,
+    SstableObjectIdManagerRef,
 };
 
 const GC_DELETE_KEYS_FOR_FLUSH: bool = false;
@@ -54,7 +55,8 @@ const GC_WATERMARK_FOR_FLUSH: u64 = 0;
 
 /// Flush shared buffer to level0. Resulted SSTs are grouped by compaction group.
 pub async fn compact(
-    context: Arc<CompactorContext>,
+    context: CompactorContext,
+    sstable_object_id_manager: SstableObjectIdManagerRef,
     payload: UploadTaskPayload,
     compaction_group_index: Arc<HashMap<TableId, CompactionGroupId>>,
 ) -> HummockResult<Vec<LocalSstableInfo>> {
@@ -81,7 +83,12 @@ pub async fn compact(
     for (id, group_payload) in grouped_payload {
         let id_copy = id;
         futures.push(
-            compact_shared_buffer(context.clone(), group_payload).map_ok(move |results| {
+            compact_shared_buffer(
+                context.clone(),
+                sstable_object_id_manager.clone(),
+                group_payload,
+            )
+            .map_ok(move |results| {
                 results
                     .into_iter()
                     .map(move |mut result| {
@@ -103,7 +110,8 @@ pub async fn compact(
 
 /// For compaction from shared buffer to level 0, this is the only function gets called.
 async fn compact_shared_buffer(
-    context: Arc<CompactorContext>,
+    context: CompactorContext,
+    sstable_object_id_manager: SstableObjectIdManagerRef,
     mut payload: UploadTaskPayload,
 ) -> HummockResult<Vec<LocalSstableInfo>> {
     // Local memory compaction looks at all key ranges.
@@ -230,6 +238,7 @@ async fn compact_shared_buffer(
             sub_compaction_sstable_size as usize,
             split_weight_by_vnode as u32,
             use_block_based_filter,
+            Box::new(sstable_object_id_manager.clone()),
         );
         let iter = OrderedMergeIteratorInner::new(
             payload.iter().map(|imm| imm.clone().into_forward_iter()),
@@ -444,10 +453,11 @@ impl SharedBufferCompactRunner {
     pub fn new(
         split_index: usize,
         key_range: KeyRange,
-        context: Arc<CompactorContext>,
+        context: CompactorContext,
         sub_compaction_sstable_size: usize,
         split_weight_by_vnode: u32,
         use_block_based_filter: bool,
+        object_id_getter: Box<dyn GetObjectId>,
     ) -> Self {
         let mut options: SstableBuilderOptions = context.storage_opts.as_ref().into();
         options.capacity = sub_compaction_sstable_size;
@@ -466,6 +476,7 @@ impl SharedBufferCompactRunner {
                 split_weight_by_vnode,
                 use_block_based_filter,
             },
+            object_id_getter,
         );
         Self {
             compactor,
diff --git a/src/storage/src/hummock/event_handler/cache_refill_policy.rs b/src/storage/src/hummock/event_handler/cache_refill_policy.rs
deleted file mode 100644
index 4b4deb1e41e4f..0000000000000
--- a/src/storage/src/hummock/event_handler/cache_refill_policy.rs
+++ /dev/null
@@ -1,277 +0,0 @@
-// Copyright 2023 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashSet;
-use std::sync::Arc;
-use std::time::{Duration, Instant};
-
-use futures::future::try_join_all;
-use itertools::Itertools;
-use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo;
-use risingwave_hummock_sdk::HummockSstableObjectId;
-use tokio::sync::{mpsc, Mutex};
-
-use crate::hummock::sstable_store::SstableStoreRef;
-use crate::hummock::{HummockError, HummockResult, TableHolder};
-use crate::monitor::{CompactorMetrics, StoreLocalStatistic};
-
-const REFILL_DATA_FILE_CACHE_CONCURRENCY: usize = 100;
-const REFILL_DATA_FILE_CACHE_TIMEOUT: Duration = Duration::from_secs(10);
-
-struct CacheRefillLevel {
-    level_idx: u32,
-    insert_sst_infos: Vec<TableHolder>,
-    delete_sst_object_ids: Vec<HummockSstableObjectId>,
-}
-
-pub struct CacheRefillPolicyConfig {
-    pub sstable_store: SstableStoreRef,
-    pub metrics: Arc<CompactorMetrics>,
-
-    pub max_preload_wait_time_mill: u64,
-
-    pub refill_data_file_cache_levels: HashSet<u32>,
-}
-
-pub struct CacheRefillPolicy {
-    sstable_store: SstableStoreRef,
-    metrics: Arc<CompactorMetrics>,
-
-    max_preload_wait_time_mill: u64,
-
-    refill_data_file_cache_levels: HashSet<u32>,
-
-    concurrency: Arc<Concurrency>,
-}
-
-impl CacheRefillPolicy {
-    pub fn new(config: CacheRefillPolicyConfig) -> Self {
-        Self {
-            sstable_store: config.sstable_store,
-            metrics: config.metrics,
-
-            max_preload_wait_time_mill: config.max_preload_wait_time_mill,
-
-            refill_data_file_cache_levels: config.refill_data_file_cache_levels,
-
-            concurrency: Arc::new(Concurrency::new(REFILL_DATA_FILE_CACHE_CONCURRENCY)),
-        }
-    }
-
-    pub async fn execute(self: &Arc<Self>, sst_delta_infos: Vec<SstDeltaInfo>, max_level: u32) {
-        if self.max_preload_wait_time_mill > 0 && !sst_delta_infos.is_empty() {
-            let policy = self.clone();
-            let handle = tokio::spawn(async move {
-                let timer = policy.metrics.refill_cache_duration.start_timer();
-                let mut preload_count = 0;
-                let stats = StoreLocalStatistic::default();
-
-                let mut reqs = vec![];
-                let mut level_idxs = vec![];
-                let mut delete_sst_object_id_levels = vec![];
-
-                for sst_delta_info in &sst_delta_infos {
-                    if sst_delta_info.insert_sst_level >= max_level
-                        || sst_delta_info.insert_sst_infos.is_empty()
-                    {
-                        continue;
-                    }
-
-                    let mut level_reqs = vec![];
-
-                    for sst_info in &sst_delta_info.insert_sst_infos {
-                        level_reqs.push(policy.sstable_store.sstable_syncable(sst_info, &stats));
-                        preload_count += 1;
-                    }
-
-                    level_idxs.push(sst_delta_info.insert_sst_level);
-                    delete_sst_object_id_levels.push(sst_delta_info.delete_sst_object_ids.clone());
-                    reqs.push(level_reqs);
-                }
-
-                policy.metrics.preload_io_count.inc_by(preload_count as u64);
-
-                let res = try_join_all(reqs.into_iter().map(try_join_all)).await;
-
-                let cache_refill_levels = match async move {
-                    let cache_refill_levels = res?
-                        .into_iter()
-                        .zip_eq_fast(level_idxs)
-                        .zip_eq_fast(delete_sst_object_id_levels)
-                        .map(|((vres, level_idx), delete_sst_object_ids)| {
-                            let insert_sst_infos = vres
-                                .into_iter()
-                                .map(|(sst_info, _, _)| sst_info)
-                                .collect_vec();
-                            CacheRefillLevel {
-                                level_idx,
-                                insert_sst_infos,
-                                delete_sst_object_ids,
-                            }
-                        })
-                        .collect_vec();
-
-                    Ok::<_, HummockError>(cache_refill_levels)
-                }
-                .await
-                {
-                    Ok(cache_refill_levels) => cache_refill_levels,
-                    Err(e) => {
-                        tracing::warn!("fill meta cache error: {:?}", e);
-                        return;
-                    }
-                };
-
-                if !cache_refill_levels.is_empty()
-                    && policy.sstable_store.data_file_cache().is_filter_enabled()
-                {
-                    tokio::spawn({
-                        async move {
-                            if let Err(e) =
-                                Self::refill_data_file_cache(policy, cache_refill_levels).await
-                            {
-                                tracing::warn!("fill data file cache error: {:?}", e);
-                            }
-                        }
-                    });
-                }
-                timer.observe_duration();
-            });
-            let _ = tokio::time::timeout(
-                Duration::from_millis(self.max_preload_wait_time_mill),
-                handle,
-            )
-            .await;
-        }
-    }
-
-    async fn refill_data_file_cache(
-        self: Arc<Self>,
-        cache_refill_levels: Vec<CacheRefillLevel>,
-    ) -> HummockResult<()> {
-        let mut handles = vec![];
-        let filter = self.sstable_store.data_file_cache_refill_filter().unwrap();
-
-        let start = Instant::now();
-
-        for CacheRefillLevel {
-            level_idx,
-            insert_sst_infos,
-            delete_sst_object_ids,
-        } in cache_refill_levels
-        {
-            if insert_sst_infos.is_empty()
-                || delete_sst_object_ids.is_empty()
-                || !self.refill_data_file_cache_levels.contains(&level_idx)
-            {
-                continue;
-            }
-
-            let blocks = insert_sst_infos
-                .iter()
-                .map(|meta| meta.value().block_count())
-                .sum::<usize>();
-
-            let mut refill = false;
-            for id in &delete_sst_object_ids {
-                if filter.contains(id) {
-                    refill = true;
-                    break;
-                }
-            }
-
-            if refill {
-                for sst_info in &insert_sst_infos {
-                    for block_index in 0..sst_info.value().block_count() {
-                        let concurrency = self.concurrency.clone();
-                        let meta = sst_info.value().clone();
-                        let mut stat = StoreLocalStatistic::default();
-                        let sstable_store = self.sstable_store.clone();
-                        let metrics = self.metrics.clone();
-
-                        concurrency.acquire().await;
-                        if start.elapsed() > REFILL_DATA_FILE_CACHE_TIMEOUT {
-                            self.metrics
-                                .refill_data_file_cache_count
-                                .with_label_values(&["timeout"])
-                                .inc();
-                            continue;
-                        }
-
-                        let future = async move {
-                            let res = sstable_store
-                                .may_fill_data_file_cache(&meta, block_index, &mut stat)
-                                .await;
-                            match res {
-                                Ok(true) => {
-                                    metrics
-                                        .refill_data_file_cache_count
-                                        .with_label_values(&["admitted"])
-                                        .inc();
-                                }
-                                Ok(false) => {
-                                    metrics
-                                        .refill_data_file_cache_count
-                                        .with_label_values(&["rejected"])
-                                        .inc();
-                                }
-                                _ => {}
-                            }
-                            concurrency.release();
-                            res
-                        };
-                        let handle = tokio::spawn(future);
-                        handles.push(handle);
-                    }
-                }
-            } else {
-                self.metrics
-                    .refill_data_file_cache_count
-                    .with_label_values(&["filtered"])
-                    .inc_by(blocks as f64);
-            }
-        }
-
-        let _ = try_join_all(handles).await;
-
-        Ok(())
-    }
-}
-
-pub struct Concurrency {
-    tx: mpsc::UnboundedSender<()>,
-    rx: Mutex<mpsc::UnboundedReceiver<()>>,
-}
-
-impl Concurrency {
-    pub fn new(concurrency: usize) -> Self {
-        let (tx, rx) = mpsc::unbounded_channel();
-        for _ in 0..concurrency {
-            tx.send(()).unwrap();
-        }
-        Self {
-            tx,
-            rx: Mutex::new(rx),
-        }
-    }
-
-    pub async fn acquire(&self) {
-        self.rx.lock().await.recv().await.unwrap();
-    }
-
-    pub fn release(&self) {
-        self.tx.send(()).unwrap();
-    }
-}
diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
index 622337561ecc7..31899737f4ab8 100644
--- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs
+++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
@@ -12,14 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap};
 use std::ops::DerefMut;
+use std::pin::pin;
 use std::sync::Arc;
 
 use arc_swap::ArcSwap;
 use await_tree::InstrumentAwait;
-use futures::future::{select, Either};
-use futures::FutureExt;
 use parking_lot::RwLock;
 use prometheus::core::{AtomicU64, GenericGauge};
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockVersionUpdateExt;
@@ -29,11 +28,11 @@ use tokio::spawn;
 use tokio::sync::{mpsc, oneshot};
 use tracing::{error, info, trace, warn};
 
-use super::cache_refill_policy::CacheRefillPolicyConfig;
+use super::refiller::{CacheRefillConfig, CacheRefiller};
 use super::{LocalInstanceGuard, LocalInstanceId, ReadVersionMappingType};
 use crate::hummock::compactor::{compact, CompactorContext};
 use crate::hummock::conflict_detector::ConflictDetector;
-use crate::hummock::event_handler::cache_refill_policy::CacheRefillPolicy;
+use crate::hummock::event_handler::refiller::CacheRefillerEvent;
 use crate::hummock::event_handler::uploader::{
     HummockUploader, UploadTaskInfo, UploadTaskPayload, UploaderEvent,
 };
@@ -44,7 +43,7 @@ use crate::hummock::store::version::{
 };
 use crate::hummock::utils::validate_table_key_range;
 use crate::hummock::{
-    HummockError, HummockResult, MemoryLimiter, SstableObjectIdManagerRef, TrackerId,
+    HummockError, HummockResult, MemoryLimiter, SstableObjectIdManager, TrackerId,
 };
 use crate::monitor::HummockStateStoreMetrics;
 use crate::opts::StorageOpts;
@@ -123,30 +122,35 @@ pub struct HummockEventHandler {
     write_conflict_detector: Option<Arc<ConflictDetector>>,
 
     uploader: HummockUploader,
+    refiller: CacheRefiller,
 
     last_instance_id: LocalInstanceId,
 
-    cache_refill_policy: Arc<CacheRefillPolicy>,
-    sstable_object_id_manager: SstableObjectIdManagerRef,
+    sstable_object_id_manager: Arc<SstableObjectIdManager>,
 }
 
 async fn flush_imms(
     payload: UploadTaskPayload,
     task_info: UploadTaskInfo,
-    compactor_context: Arc<crate::hummock::compactor::CompactorContext>,
+    compactor_context: CompactorContext,
+    sstable_object_id_manager: Arc<SstableObjectIdManager>,
 ) -> HummockResult<Vec<LocalSstableInfo>> {
     for epoch in &task_info.epochs {
-        let _ = compactor_context
-            .sstable_object_id_manager
+        let _ = sstable_object_id_manager
             .add_watermark_object_id(Some(*epoch))
             .await
             .inspect_err(|e| {
                 error!("unable to set watermark sst id. epoch: {}, {:?}", epoch, e);
             });
     }
-    compact(compactor_context, payload, task_info.compaction_group_index)
-        .verbose_instrument_await("shared_buffer_compact")
-        .await
+    compact(
+        compactor_context,
+        sstable_object_id_manager,
+        payload,
+        task_info.compaction_group_index,
+    )
+    .verbose_instrument_await("shared_buffer_compact")
+    .await
 }
 
 impl HummockEventHandler {
@@ -154,9 +158,10 @@ impl HummockEventHandler {
         hummock_event_tx: mpsc::UnboundedSender<HummockEvent>,
         hummock_event_rx: mpsc::UnboundedReceiver<HummockEvent>,
         pinned_version: PinnedVersion,
-        compactor_context: Arc<CompactorContext>,
+        compactor_context: CompactorContext,
+        sstable_object_id_manager: Arc<SstableObjectIdManager>,
         state_store_metrics: Arc<HummockStateStoreMetrics>,
-        refill_data_file_cache_levels: HashSet<u32>,
+        cache_refill_config: CacheRefillConfig,
     ) -> Self {
         let (version_update_notifier_tx, _) =
             tokio::sync::watch::channel(pinned_version.max_committed_epoch());
@@ -166,13 +171,11 @@ impl HummockEventHandler {
             &compactor_context.storage_opts,
             state_store_metrics.uploader_uploading_task_size.clone(),
         );
-        let max_preload_wait_time_mill = compactor_context.storage_opts.max_preload_wait_time_mill;
         let write_conflict_detector =
             ConflictDetector::new_from_config(&compactor_context.storage_opts);
         let sstable_store = compactor_context.sstable_store.clone();
-        let metrics = compactor_context.compactor_metrics.clone();
-        let sstable_object_id_manager = compactor_context.sstable_object_id_manager.clone();
         let upload_compactor_context = compactor_context.clone();
+        let cloned_sstable_object_id_manager = sstable_object_id_manager.clone();
         let uploader = HummockUploader::new(
             state_store_metrics,
             pinned_version.clone(),
@@ -181,19 +184,14 @@ impl HummockEventHandler {
                     payload,
                     task_info,
                     upload_compactor_context.clone(),
+                    cloned_sstable_object_id_manager.clone(),
                 ))
             }),
             buffer_tracker,
             &compactor_context.storage_opts,
             compactor_context.compaction_executor.clone(),
         );
-        let cache_refill_policy_config = CacheRefillPolicyConfig {
-            sstable_store,
-            metrics,
-            max_preload_wait_time_mill,
-            refill_data_file_cache_levels,
-        };
-        let cache_refill_policy = Arc::new(CacheRefillPolicy::new(cache_refill_policy_config));
+        let refiller = CacheRefiller::new(cache_refill_config, sstable_store);
 
         Self {
             hummock_event_tx,
@@ -204,8 +202,8 @@ impl HummockEventHandler {
             write_conflict_detector,
             read_version_mapping,
             uploader,
+            refiller,
             last_instance_id: 0,
-            cache_refill_policy,
             sstable_object_id_manager,
         }
     }
@@ -227,20 +225,6 @@ impl HummockEventHandler {
     }
 }
 
-impl HummockEventHandler {
-    async fn next_event(&mut self) -> Option<Either<UploaderEvent, HummockEvent>> {
-        match select(
-            self.uploader.next_event(),
-            self.hummock_event_rx.recv().boxed(),
-        )
-        .await
-        {
-            Either::Left((event, _)) => Some(Either::Left(event)),
-            Either::Right((event, _)) => event.map(Either::Right),
-        }
-    }
-}
-
 // Handler for different events
 impl HummockEventHandler {
     fn handle_epoch_synced(
@@ -414,29 +398,26 @@ impl HummockEventHandler {
         });
     }
 
-    async fn handle_version_update(&mut self, version_payload: Payload) {
-        let pinned_version = self.pinned_version.load().clone();
+    fn handle_version_update(&mut self, version_payload: Payload) {
+        let pinned_version = self
+            .refiller
+            .last_new_pinned_version()
+            .cloned()
+            .map(Arc::new)
+            .unwrap_or_else(|| self.pinned_version.load().clone());
 
-        let prev_max_committed_epoch = pinned_version.max_committed_epoch();
+        let mut sst_delta_infos = vec![];
         let newly_pinned_version = match version_payload {
             Payload::VersionDeltas(version_deltas) => {
                 let mut version_to_apply = pinned_version.version();
-                let max_level = version_to_apply
-                    .levels
-                    .values()
-                    .map(|levels| levels.levels.last().unwrap().level_idx)
-                    .max()
-                    .unwrap();
                 for version_delta in &version_deltas.version_deltas {
                     assert_eq!(version_to_apply.id, version_delta.prev_id);
                     if version_to_apply.max_committed_epoch == version_delta.max_committed_epoch {
-                        let sst_delta_infos = version_to_apply.build_sst_delta_infos(version_delta);
-                        self.cache_refill_policy
-                            .execute(sst_delta_infos, max_level)
-                            .await;
+                        sst_delta_infos = version_to_apply.build_sst_delta_infos(version_delta);
                     }
                     version_to_apply.apply_version_delta(version_delta);
                 }
+
                 version_to_apply
             }
             Payload::PinnedVersion(version) => version,
@@ -445,6 +426,16 @@ impl HummockEventHandler {
         validate_table_key_range(&newly_pinned_version);
 
         let new_pinned_version = pinned_version.new_pin_version(newly_pinned_version);
+
+        self.refiller
+            .start_cache_refill(sst_delta_infos, pinned_version, new_pinned_version);
+    }
+
+    fn apply_version_update(
+        &mut self,
+        pinned_version: Arc<PinnedVersion>,
+        new_pinned_version: PinnedVersion,
+    ) {
         self.pinned_version
             .store(Arc::new(new_pinned_version.clone()));
 
@@ -454,6 +445,7 @@ impl HummockEventHandler {
             });
         }
 
+        let prev_max_committed_epoch = pinned_version.max_committed_epoch();
         let max_committed_epoch = new_pinned_version.max_committed_epoch();
 
         // only notify local_version_manager when MCE change
@@ -487,158 +479,184 @@ impl HummockEventHandler {
 
 impl HummockEventHandler {
     pub async fn start_hummock_event_handler_worker(mut self) {
-        while let Some(event) = self.next_event().await {
-            match event {
-                Either::Left(event) => match event {
-                    UploaderEvent::SyncFinish(epoch, newly_uploaded_sstables) => {
-                        self.handle_epoch_synced(epoch, newly_uploaded_sstables);
+        loop {
+            tokio::select! {
+                event = self.uploader.next_event() => {
+                    self.handle_uploader_event(event);
+                }
+                event = self.refiller.next_event() => {
+                    let CacheRefillerEvent {pinned_version, new_pinned_version } = event;
+                    self.apply_version_update(pinned_version, new_pinned_version);
+                }
+                event = pin!(self.hummock_event_rx.recv()) => {
+                    let Some(event) = event else { break };
+                    if self.handle_hummock_event(event) {
+                        break;
                     }
+                }
+            }
+        }
+    }
 
-                    UploaderEvent::DataSpilled(staging_sstable_info) => {
-                        self.handle_data_spilled(staging_sstable_info);
-                    }
+    fn handle_uploader_event(&mut self, event: UploaderEvent) {
+        match event {
+            UploaderEvent::SyncFinish(epoch, newly_uploaded_sstables) => {
+                self.handle_epoch_synced(epoch, newly_uploaded_sstables);
+            }
+
+            UploaderEvent::DataSpilled(staging_sstable_info) => {
+                self.handle_data_spilled(staging_sstable_info);
+            }
 
-                    UploaderEvent::ImmMerged(merge_output) => {
-                        // update read version for corresponding table shards
-                        let read_guard = self.read_version_mapping.read();
-                        if let Some(shards) = read_guard.get(&merge_output.table_id) {
-                            shards.get(&merge_output.instance_id).map_or_else(
-                                || {
-                                    warn!(
-                                        "handle ImmMerged: table instance not found. table {}, instance {}",
-                                        &merge_output.table_id, &merge_output.instance_id
-                                    )
-                                },
-                                |read_version| {
-                                    read_version.write().update(VersionUpdate::Staging(
-                                        StagingData::MergedImmMem(merge_output.merged_imm),
-                                    ));
-                                },
+            UploaderEvent::ImmMerged(merge_output) => {
+                // update read version for corresponding table shards
+                let read_guard = self.read_version_mapping.read();
+                read_guard.get(&merge_output.table_id).map_or((), |shards| {
+                    shards.get(&merge_output.instance_id).map_or_else(
+                        || {
+                            warn!(
+                                "handle ImmMerged: table instance not found. table {}, instance {}",
+                                &merge_output.table_id, &merge_output.instance_id
                             )
-                        };
-                    }
-                },
-                Either::Right(event) => {
-                    match event {
-                        HummockEvent::BufferMayFlush => {
-                            self.uploader.may_flush();
-                        }
-                        HummockEvent::AwaitSyncEpoch {
-                            new_sync_epoch,
-                            sync_result_sender,
-                        } => {
-                            self.handle_await_sync_epoch(new_sync_epoch, sync_result_sender);
-                        }
-                        HummockEvent::Clear(notifier) => {
-                            self.handle_clear(notifier);
-                        }
-                        HummockEvent::Shutdown => {
-                            info!("buffer tracker shutdown");
-                            break;
-                        }
-
-                        HummockEvent::VersionUpdate(version_payload) => {
-                            self.handle_version_update(version_payload).await;
-                        }
-
-                        HummockEvent::ImmToUploader(imm) => {
-                            self.uploader.add_imm(imm);
-                            self.uploader.may_flush();
-                        }
-
-                        HummockEvent::SealEpoch {
-                            epoch,
-                            is_checkpoint,
-                        } => {
-                            self.uploader.seal_epoch(epoch);
-
-                            if is_checkpoint {
-                                self.uploader.start_sync_epoch(epoch);
-                            } else {
-                                // start merging task on non-checkpoint epochs sealed
-                                self.uploader.start_merge_imms(epoch);
-                            }
-                        }
-                        #[cfg(any(test, feature = "test"))]
-                        HummockEvent::FlushEvent(sender) => {
-                            let _ = sender.send(()).inspect_err(|e| {
-                                error!("unable to send flush result: {:?}", e);
-                            });
-                        }
-
-                        HummockEvent::RegisterReadVersion {
-                            table_id,
-                            new_read_version_sender,
-                            is_replicated,
-                        } => {
-                            let pinned_version = self.pinned_version.load();
-                            let basic_read_version = Arc::new(RwLock::new(
-                                HummockReadVersion::new_with_replication_option(
-                                    (**pinned_version).clone(),
-                                    is_replicated,
-                                ),
+                        },
+                        |read_version| {
+                            read_version.write().update(VersionUpdate::Staging(
+                                StagingData::MergedImmMem(merge_output.merged_imm),
                             ));
+                        },
+                    )
+                });
+            }
+        }
+    }
+
+    /// Gracefully shutdown if returns `true`.
+    fn handle_hummock_event(&mut self, event: HummockEvent) -> bool {
+        match event {
+            HummockEvent::BufferMayFlush => {
+                self.uploader.may_flush();
+            }
+            HummockEvent::AwaitSyncEpoch {
+                new_sync_epoch,
+                sync_result_sender,
+            } => {
+                self.handle_await_sync_epoch(new_sync_epoch, sync_result_sender);
+            }
+            HummockEvent::Clear(notifier) => {
+                self.handle_clear(notifier);
+            }
+            HummockEvent::Shutdown => {
+                info!("buffer tracker shutdown");
+                return true;
+            }
 
-                            let instance_id = self.generate_instance_id();
-
-                            info_in_release!(
-                                "new read version registered: table_id: {}, instance_id: {}",
-                                table_id,
-                                instance_id
-                            );
-
-                            {
-                                let mut read_version_mapping_guard =
-                                    self.read_version_mapping.write();
-
-                                read_version_mapping_guard
-                                    .entry(table_id)
-                                    .or_default()
-                                    .insert(instance_id, basic_read_version.clone());
-                            }
-
-                            match new_read_version_sender.send((
-                                basic_read_version,
-                                LocalInstanceGuard {
-                                    table_id,
-                                    instance_id,
-                                    event_sender: self.hummock_event_tx.clone(),
-                                },
-                            )) {
-                                Ok(_) => {}
-                                Err(_) => {
-                                    panic!("RegisterReadVersion send fail table_id {:?} instance_is {:?}", table_id, instance_id)
-                                }
-                            }
-                        }
-
-                        HummockEvent::DestroyReadVersion {
-                            table_id,
-                            instance_id,
-                        } => {
-                            info_in_release!(
-                                "read version deregister: table_id: {}, instance_id: {}",
-                                table_id,
-                                instance_id
-                            );
-                            let mut read_version_mapping_guard = self.read_version_mapping.write();
-                            let entry = read_version_mapping_guard
-                                .get_mut(&table_id)
-                                .unwrap_or_else(|| {
-                                    panic!(
-                                        "DestroyHummockInstance table_id {} instance_id {} fail",
-                                        table_id, instance_id
-                                    )
-                                });
-                            entry.remove(&instance_id).unwrap_or_else(|| panic!("DestroyHummockInstance inexist instance table_id {} instance_id {}",  table_id, instance_id));
-                            if entry.is_empty() {
-                                read_version_mapping_guard.remove(&table_id);
-                            }
-                        }
+            HummockEvent::VersionUpdate(version_payload) => {
+                self.handle_version_update(version_payload);
+            }
+
+            HummockEvent::ImmToUploader(imm) => {
+                self.uploader.add_imm(imm);
+                self.uploader.may_flush();
+            }
+
+            HummockEvent::SealEpoch {
+                epoch,
+                is_checkpoint,
+            } => {
+                self.uploader.seal_epoch(epoch);
+
+                if is_checkpoint {
+                    self.uploader.start_sync_epoch(epoch);
+                } else {
+                    // start merging task on non-checkpoint epochs sealed
+                    self.uploader.start_merge_imms(epoch);
+                }
+            }
+            #[cfg(any(test, feature = "test"))]
+            HummockEvent::FlushEvent(sender) => {
+                let _ = sender.send(()).inspect_err(|e| {
+                    error!("unable to send flush result: {:?}", e);
+                });
+            }
+
+            HummockEvent::RegisterReadVersion {
+                table_id,
+                new_read_version_sender,
+                is_replicated,
+            } => {
+                let pinned_version = self.pinned_version.load();
+                let basic_read_version = Arc::new(RwLock::new(
+                    HummockReadVersion::new_with_replication_option(
+                        (**pinned_version).clone(),
+                        is_replicated,
+                    ),
+                ));
+
+                let instance_id = self.generate_instance_id();
+
+                info_in_release!(
+                    "new read version registered: table_id: {}, instance_id: {}",
+                    table_id,
+                    instance_id
+                );
+
+                {
+                    let mut read_version_mapping_guard = self.read_version_mapping.write();
+
+                    read_version_mapping_guard
+                        .entry(table_id)
+                        .or_default()
+                        .insert(instance_id, basic_read_version.clone());
+                }
+
+                match new_read_version_sender.send((
+                    basic_read_version,
+                    LocalInstanceGuard {
+                        table_id,
+                        instance_id,
+                        event_sender: self.hummock_event_tx.clone(),
+                    },
+                )) {
+                    Ok(_) => {}
+                    Err(_) => {
+                        panic!(
+                            "RegisterReadVersion send fail table_id {:?} instance_is {:?}",
+                            table_id, instance_id
+                        )
                     }
                 }
-            };
+            }
+
+            HummockEvent::DestroyReadVersion {
+                table_id,
+                instance_id,
+            } => {
+                info_in_release!(
+                    "read version deregister: table_id: {}, instance_id: {}",
+                    table_id,
+                    instance_id
+                );
+                let mut read_version_mapping_guard = self.read_version_mapping.write();
+                let entry = read_version_mapping_guard
+                    .get_mut(&table_id)
+                    .unwrap_or_else(|| {
+                        panic!(
+                            "DestroyHummockInstance table_id {} instance_id {} fail",
+                            table_id, instance_id
+                        )
+                    });
+                entry.remove(&instance_id).unwrap_or_else(|| {
+                    panic!(
+                        "DestroyHummockInstance inexist instance table_id {} instance_id {}",
+                        table_id, instance_id
+                    )
+                });
+                if entry.is_empty() {
+                    read_version_mapping_guard.remove(&table_id);
+                }
+            }
         }
+        false
     }
 
     fn generate_instance_id(&mut self) -> LocalInstanceId {
diff --git a/src/storage/src/hummock/event_handler/mod.rs b/src/storage/src/hummock/event_handler/mod.rs
index 20ffe0cea0783..7b9bfd09835cd 100644
--- a/src/storage/src/hummock/event_handler/mod.rs
+++ b/src/storage/src/hummock/event_handler/mod.rs
@@ -26,8 +26,8 @@ use crate::hummock::store::memtable::ImmutableMemtable;
 use crate::hummock::HummockResult;
 use crate::store::SyncResult;
 
-mod cache_refill_policy;
 pub mod hummock_event_handler;
+pub mod refiller;
 pub mod uploader;
 
 pub use hummock_event_handler::HummockEventHandler;
diff --git a/src/storage/src/hummock/event_handler/refiller.rs b/src/storage/src/hummock/event_handler/refiller.rs
new file mode 100644
index 0000000000000..7e14f92bd76a4
--- /dev/null
+++ b/src/storage/src/hummock/event_handler/refiller.rs
@@ -0,0 +1,314 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashSet, VecDeque};
+use std::ops::DerefMut;
+use std::pin::Pin;
+use std::sync::{Arc, LazyLock};
+use std::task::{ready, Context, Poll};
+use std::time::{Duration, Instant};
+
+use futures::future::{join_all, try_join_all};
+use futures::{Future, FutureExt};
+use itertools::Itertools;
+use prometheus::core::{AtomicU64, GenericCounter};
+use prometheus::{
+    register_histogram_vec_with_registry, register_histogram_with_registry,
+    register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, HistogramVec,
+    IntGauge, Registry,
+};
+use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo;
+use tokio::sync::Semaphore;
+use tokio::task::JoinHandle;
+
+use crate::hummock::local_version::pinned_version::PinnedVersion;
+use crate::hummock::{HummockResult, SstableStoreRef, TableHolder};
+use crate::monitor::StoreLocalStatistic;
+
+pub static GLOBAL_CACHE_REFILL_METRICS: LazyLock<CacheRefillMetrics> =
+    LazyLock::new(|| CacheRefillMetrics::new(&GLOBAL_METRICS_REGISTRY));
+
+pub struct CacheRefillMetrics {
+    pub data_refill_duration: HistogramVec,
+
+    pub data_refill_duration_admitted: Histogram,
+    pub data_refill_duration_rejected: Histogram,
+    pub data_refill_filtered_total: GenericCounter<AtomicU64>,
+
+    pub meta_refill_duration: Histogram,
+
+    pub refill_queue_total: IntGauge,
+}
+
+impl CacheRefillMetrics {
+    pub fn new(registry: &Registry) -> Self {
+        let data_refill_duration = register_histogram_vec_with_registry!(
+            "data_refill_duration",
+            "data refill duration",
+            &["op"],
+            registry,
+        )
+        .unwrap();
+        let data_refill_duration_admitted = data_refill_duration
+            .get_metric_with_label_values(&["admitted"])
+            .unwrap();
+        let data_refill_duration_rejected = data_refill_duration
+            .get_metric_with_label_values(&["rejected"])
+            .unwrap();
+
+        let data_refill_filtered_total = register_int_counter_with_registry!(
+            "data_refill_filtered_total",
+            "data refill filtered total",
+            registry,
+        )
+        .unwrap();
+
+        let meta_refill_duration = register_histogram_with_registry!(
+            "meta_refill_duration",
+            "meta refill duration",
+            registry,
+        )
+        .unwrap();
+
+        let refill_queue_total = register_int_gauge_with_registry!(
+            "refill_queue_total",
+            "refill queue total",
+            registry,
+        )
+        .unwrap();
+
+        Self {
+            data_refill_duration,
+            data_refill_duration_admitted,
+            data_refill_duration_rejected,
+            data_refill_filtered_total,
+            meta_refill_duration,
+            refill_queue_total,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct CacheRefillConfig {
+    pub timeout: Duration,
+    pub data_refill_levels: HashSet<u32>,
+    pub concurrency: usize,
+}
+
+struct Item {
+    handle: JoinHandle<()>,
+    event: CacheRefillerEvent,
+}
+
+/// A cache refiller for hummock data.
+pub struct CacheRefiller {
+    /// order: old => new
+    queue: VecDeque<Item>,
+
+    context: CacheRefillContext,
+}
+
+impl CacheRefiller {
+    pub fn new(config: CacheRefillConfig, sstable_store: SstableStoreRef) -> Self {
+        let config = Arc::new(config);
+        let concurrency = Arc::new(Semaphore::new(config.concurrency));
+        Self {
+            queue: VecDeque::new(),
+            context: CacheRefillContext {
+                config,
+                concurrency,
+                sstable_store,
+            },
+        }
+    }
+
+    pub fn start_cache_refill(
+        &mut self,
+        deltas: Vec<SstDeltaInfo>,
+        pinned_version: Arc<PinnedVersion>,
+        new_pinned_version: PinnedVersion,
+    ) {
+        let task = CacheRefillTask {
+            deltas,
+            context: self.context.clone(),
+        };
+        let event = CacheRefillerEvent {
+            pinned_version,
+            new_pinned_version,
+        };
+        let handle = tokio::spawn(task.run());
+        let item = Item { handle, event };
+        self.queue.push_back(item);
+        GLOBAL_CACHE_REFILL_METRICS.refill_queue_total.add(1);
+    }
+
+    pub fn last_new_pinned_version(&self) -> Option<&PinnedVersion> {
+        self.queue.back().map(|item| &item.event.new_pinned_version)
+    }
+
+    pub fn next_event(&mut self) -> NextCacheRefillerEvent<'_> {
+        NextCacheRefillerEvent { refiller: self }
+    }
+}
+
+pub struct NextCacheRefillerEvent<'a> {
+    refiller: &'a mut CacheRefiller,
+}
+
+impl<'a> Future for NextCacheRefillerEvent<'a> {
+    type Output = CacheRefillerEvent;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let refiller = &mut self.deref_mut().refiller;
+
+        if let Some(item) = refiller.queue.front_mut() {
+            ready!(item.handle.poll_unpin(cx)).unwrap();
+            let item = refiller.queue.pop_front().unwrap();
+            GLOBAL_CACHE_REFILL_METRICS.refill_queue_total.sub(1);
+            return Poll::Ready(item.event);
+        }
+        Poll::Pending
+    }
+}
+
+pub struct CacheRefillerEvent {
+    pub pinned_version: Arc<PinnedVersion>,
+    pub new_pinned_version: PinnedVersion,
+}
+
+#[derive(Clone)]
+struct CacheRefillContext {
+    config: Arc<CacheRefillConfig>,
+    concurrency: Arc<Semaphore>,
+    sstable_store: SstableStoreRef,
+}
+
+pub struct CacheRefillTask {
+    deltas: Vec<SstDeltaInfo>,
+    context: CacheRefillContext,
+}
+
+impl CacheRefillTask {
+    async fn run(self) {
+        let tasks = self
+            .deltas
+            .iter()
+            .map(|delta| {
+                let context = self.context.clone();
+                async move {
+                    let holders = match Self::meta_cache_refill(&context, delta).await {
+                        Ok(holders) => holders,
+                        Err(e) => {
+                            tracing::warn!("meta cache refill error: {:?}", e);
+                            return;
+                        }
+                    };
+                    Self::data_cache_refill(&context, delta, holders).await;
+                }
+            })
+            .collect_vec();
+        let future = join_all(tasks);
+
+        let _ = tokio::time::timeout(self.context.config.timeout, future).await;
+    }
+
+    async fn meta_cache_refill(
+        context: &CacheRefillContext,
+        delta: &SstDeltaInfo,
+    ) -> HummockResult<Vec<TableHolder>> {
+        let stats = StoreLocalStatistic::default();
+        let tasks = delta
+            .insert_sst_infos
+            .iter()
+            .map(|info| async {
+                let _timer = GLOBAL_CACHE_REFILL_METRICS
+                    .meta_refill_duration
+                    .start_timer();
+                context.sstable_store.sstable_syncable(info, &stats).await
+            })
+            .collect_vec();
+        let res = try_join_all(tasks).await?;
+        let holders = res.into_iter().map(|(holder, _, _)| holder).collect_vec();
+        Ok(holders)
+    }
+
+    async fn data_cache_refill(
+        context: &CacheRefillContext,
+        delta: &SstDeltaInfo,
+        holders: Vec<TableHolder>,
+    ) {
+        let now = Instant::now();
+
+        // return if data file cache is disabled
+        let Some(filter) = context.sstable_store.data_file_cache_refill_filter() else {
+            return;
+        };
+
+        // return if no data to refill
+        if delta.insert_sst_infos.is_empty() || delta.delete_sst_object_ids.is_empty() {
+            return;
+        }
+
+        // return if filtered
+        if !context
+            .config
+            .data_refill_levels
+            .contains(&delta.insert_sst_level)
+            || !delta
+                .delete_sst_object_ids
+                .iter()
+                .any(|id| filter.contains(id))
+        {
+            let blocks = holders
+                .iter()
+                .map(|meta| meta.value().block_count() as u64)
+                .sum();
+            GLOBAL_CACHE_REFILL_METRICS
+                .data_refill_filtered_total
+                .inc_by(blocks);
+            return;
+        }
+
+        let mut tasks = vec![];
+        for sst_info in &holders {
+            for block_index in 0..sst_info.value().block_count() {
+                let meta = sst_info.value();
+                let mut stat = StoreLocalStatistic::default();
+                let task = async move {
+                    let permit = context.concurrency.acquire().await.unwrap();
+                    match context
+                        .sstable_store
+                        .may_fill_data_file_cache(meta, block_index, &mut stat)
+                        .await
+                    {
+                        Ok(true) => GLOBAL_CACHE_REFILL_METRICS
+                            .data_refill_duration_admitted
+                            .observe(now.elapsed().as_secs_f64()),
+                        Ok(false) => GLOBAL_CACHE_REFILL_METRICS
+                            .data_refill_duration_rejected
+                            .observe(now.elapsed().as_secs_f64()),
+                        Err(e) => {
+                            tracing::warn!("data cache refill error: {:?}", e);
+                        }
+                    }
+                    drop(permit);
+                };
+                tasks.push(task);
+            }
+        }
+
+        join_all(tasks).await;
+    }
+}
diff --git a/src/storage/src/hummock/file_cache/store.rs b/src/storage/src/hummock/file_cache/store.rs
index 983c053288af5..da47c1562eaee 100644
--- a/src/storage/src/hummock/file_cache/store.rs
+++ b/src/storage/src/hummock/file_cache/store.rs
@@ -23,9 +23,9 @@ use foyer::common::code::{Key, Value};
 use foyer::storage::admission::rated_random::RatedRandomAdmissionPolicy;
 use foyer::storage::admission::AdmissionPolicy;
 use foyer::storage::event::EventListener;
-use foyer::storage::store::{FetchValueFuture, PrometheusConfig};
+pub use foyer::storage::metrics::set_metrics_registry as set_foyer_metrics_registry;
+use foyer::storage::store::FetchValueFuture;
 use foyer::storage::LfuFsStoreConfig;
-use prometheus::Registry;
 use risingwave_common::util::runtime::BackgroundShutdownRuntime;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 
@@ -59,6 +59,7 @@ where
     K: Key,
     V: Value,
 {
+    pub name: String,
     pub dir: PathBuf,
     pub capacity: usize,
     pub file_capacity: usize,
@@ -73,8 +74,6 @@ where
     pub lfu_window_to_cache_size_ratio: usize,
     pub lfu_tiny_lru_capacity_ratio: f64,
     pub rated_random_rate: usize,
-    pub prometheus_registry: Option<Registry>,
-    pub prometheus_namespace: Option<String>,
     pub event_listener: Vec<Arc<dyn EventListener<K = K, V = V>>>,
     pub enable_filter: bool,
 }
@@ -201,6 +200,7 @@ where
                 }
 
                 let c = LfuFsStoreConfig {
+                    name: foyer_store_config.name,
                     eviction_config: EvictionConfig {
                         window_to_cache_size_ratio: foyer_store_config
                             .lfu_window_to_cache_size_ratio,
@@ -222,10 +222,6 @@ where
                     reclaim_rate_limit: foyer_store_config.reclaim_rate_limit,
                     recover_concurrency: foyer_store_config.recover_concurrency,
                     event_listeners: foyer_store_config.event_listener,
-                    prometheus_config: PrometheusConfig {
-                        registry: foyer_store_config.prometheus_registry,
-                        namespace: foyer_store_config.prometheus_namespace,
-                    },
                     clean_region_threshold: foyer_store_config.reclaimers
                         + foyer_store_config.reclaimers / 2,
                 };
diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs
index 76c5951e90ac0..0e93eeff782bc 100644
--- a/src/storage/src/hummock/mod.rs
+++ b/src/storage/src/hummock/mod.rs
@@ -17,6 +17,7 @@
 use std::ops::{Bound, Deref};
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
+use std::time::Duration;
 
 use arc_swap::ArcSwap;
 use bytes::Bytes;
@@ -32,6 +33,7 @@ use tracing::log::error;
 mod block_cache;
 pub use block_cache::*;
 
+use crate::filter_key_extractor::RpcFilterKeyExtractorManager;
 use crate::hummock::store::state_store::LocalHummockStorage;
 use crate::opts::StorageOpts;
 
@@ -69,11 +71,12 @@ use risingwave_common_service::observer_manager::{NotificationClient, ObserverMa
 pub use validator::*;
 use value::*;
 
+use self::event_handler::refiller::CacheRefillConfig;
 use self::event_handler::ReadVersionMappingType;
 use self::iterator::HummockIterator;
 pub use self::sstable_store::*;
 use super::monitor::HummockStateStoreMetrics;
-use crate::filter_key_extractor::{FilterKeyExtractorManager, FilterKeyExtractorManagerRef};
+use crate::filter_key_extractor::FilterKeyExtractorManager;
 use crate::hummock::backup_reader::{BackupReader, BackupReaderRef};
 use crate::hummock::compactor::CompactorContext;
 use crate::hummock::event_handler::hummock_event_handler::BufferTracker;
@@ -104,7 +107,9 @@ impl Drop for HummockStorageShutdownGuard {
 pub struct HummockStorage {
     hummock_event_sender: UnboundedSender<HummockEvent>,
 
-    context: Arc<CompactorContext>,
+    context: CompactorContext,
+
+    sstable_object_id_manager: SstableObjectIdManagerRef,
 
     buffer_tracker: BufferTracker,
 
@@ -136,7 +141,7 @@ impl HummockStorage {
         sstable_store: SstableStoreRef,
         hummock_meta_client: Arc<dyn HummockMetaClient>,
         notification_client: impl NotificationClient,
-        filter_key_extractor_manager: Arc<FilterKeyExtractorManager>,
+        filter_key_extractor_manager: Arc<RpcFilterKeyExtractorManager>,
         state_store_metrics: Arc<HummockStateStoreMetrics>,
         compactor_metrics: Arc<CompactorMetrics>,
     ) -> HummockResult<Self> {
@@ -177,14 +182,14 @@ impl HummockStorage {
             hummock_meta_client.clone(),
         ));
 
-        let compactor_context = Arc::new(CompactorContext::new_local_compact_context(
+        let compactor_context = CompactorContext::new_local_compact_context(
             options.clone(),
             sstable_store.clone(),
-            hummock_meta_client.clone(),
             compactor_metrics.clone(),
-            sstable_object_id_manager.clone(),
-            filter_key_extractor_manager.clone(),
-        ));
+            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+                filter_key_extractor_manager.clone(),
+            ),
+        );
 
         let seal_epoch = Arc::new(AtomicU64::new(pinned_version.max_committed_epoch()));
         let min_current_epoch = Arc::new(AtomicU64::new(pinned_version.max_committed_epoch()));
@@ -193,16 +198,22 @@ impl HummockStorage {
             event_rx,
             pinned_version,
             compactor_context.clone(),
+            sstable_object_id_manager.clone(),
             state_store_metrics.clone(),
-            options
-                .data_file_cache_refill_levels
-                .iter()
-                .copied()
-                .collect(),
+            CacheRefillConfig {
+                timeout: Duration::from_millis(options.cache_refill_timeout_ms),
+                data_refill_levels: options
+                    .cache_refill_data_refill_levels
+                    .iter()
+                    .copied()
+                    .collect(),
+                concurrency: options.cache_refill_concurrency,
+            },
         );
 
         let instance = Self {
             context: compactor_context,
+            sstable_object_id_manager,
             buffer_tracker: hummock_event_handler.buffer_tracker().clone(),
             version_update_notifier_tx: hummock_event_handler.version_update_notifier_tx(),
             seal_epoch,
@@ -237,6 +248,7 @@ impl HummockStorage {
             .unwrap();
 
         let (basic_read_version, instance_guard) = rx.await.unwrap();
+        let version_update_notifier_tx = self.version_update_notifier_tx.clone();
         LocalHummockStorage::new(
             instance_guard,
             basic_read_version,
@@ -245,6 +257,7 @@ impl HummockStorage {
             self.buffer_tracker.get_memory_limiter().clone(),
             self.write_limiter.clone(),
             option,
+            version_update_notifier_tx,
         )
     }
 
@@ -253,10 +266,10 @@ impl HummockStorage {
     }
 
     pub fn sstable_object_id_manager(&self) -> &SstableObjectIdManagerRef {
-        &self.context.sstable_object_id_manager
+        &self.sstable_object_id_manager
     }
 
-    pub fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManagerRef {
+    pub fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager {
         &self.context.filter_key_extractor_manager
     }
 
@@ -327,7 +340,7 @@ impl HummockStorage {
             sstable_store,
             hummock_meta_client,
             notification_client,
-            Arc::new(FilterKeyExtractorManager::default()),
+            Arc::new(RpcFilterKeyExtractorManager::default()),
             Arc::new(HummockStateStoreMetrics::unused()),
             Arc::new(CompactorMetrics::unused()),
         )
diff --git a/src/storage/src/hummock/sstable/sstable_object_id_manager.rs b/src/storage/src/hummock/sstable/sstable_object_id_manager.rs
index d37cceb3f8bc9..6ae7ddad4a7ea 100644
--- a/src/storage/src/hummock/sstable/sstable_object_id_manager.rs
+++ b/src/storage/src/hummock/sstable/sstable_object_id_manager.rs
@@ -14,7 +14,7 @@
 
 use std::cmp;
 use std::collections::hash_map::Entry;
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::ops::DerefMut;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
@@ -28,9 +28,13 @@ use sync_point::sync_point;
 use tokio::sync::oneshot;
 
 use crate::hummock::{HummockError, HummockResult};
-
 pub type SstableObjectIdManagerRef = Arc<SstableObjectIdManager>;
-
+use dyn_clone::DynClone;
+#[async_trait::async_trait]
+pub trait GetObjectId: DynClone + Send + Sync {
+    async fn get_new_sst_object_id(&mut self) -> HummockResult<HummockSstableObjectId>;
+}
+dyn_clone::clone_trait_object!(GetObjectId);
 /// 1. Caches SST object ids fetched from meta.
 /// 2. Maintains GC watermark SST object id.
 ///
@@ -59,15 +63,6 @@ impl SstableObjectIdManager {
         }
     }
 
-    /// Returns a new SST id.
-    /// The id is guaranteed to be monotonic increasing.
-    pub async fn get_new_sst_object_id(self: &Arc<Self>) -> HummockResult<HummockSstableObjectId> {
-        self.map_next_sst_object_id(|available_sst_object_ids| {
-            available_sst_object_ids.get_next_sst_object_id()
-        })
-        .await
-    }
-
     /// Executes `f` with next SST id.
     /// May fetch new SST ids via RPC.
     async fn map_next_sst_object_id<F>(
@@ -191,6 +186,41 @@ impl SstableObjectIdManager {
     }
 }
 
+#[async_trait::async_trait]
+impl GetObjectId for Arc<SstableObjectIdManager> {
+    /// Returns a new SST id.
+    /// The id is guaranteed to be monotonic increasing.
+    async fn get_new_sst_object_id(&mut self) -> HummockResult<HummockSstableObjectId> {
+        self.map_next_sst_object_id(|available_sst_object_ids| {
+            available_sst_object_ids.get_next_sst_object_id()
+        })
+        .await
+    }
+}
+
+/// `SharedComapctorObjectIdManager` is used to get output sst id for serverless compaction.
+#[derive(Clone)]
+pub struct SharedComapctorObjectIdManager {
+    output_object_ids: VecDeque<u64>,
+}
+
+impl SharedComapctorObjectIdManager {
+    pub fn new(output_object_ids: VecDeque<u64>) -> Self {
+        Self { output_object_ids }
+    }
+}
+
+#[async_trait::async_trait]
+impl GetObjectId for SharedComapctorObjectIdManager {
+    async fn get_new_sst_object_id(&mut self) -> HummockResult<HummockSstableObjectId> {
+        if let Some(first_element) = self.output_object_ids.pop_front() {
+            Ok(first_element)
+        } else {
+            return Err(HummockError::other("Output object id runs out"));
+        }
+    }
+}
+
 #[async_trait::async_trait]
 impl ExtraInfoSource for SstableObjectIdManager {
     async fn get_extra_info(&self) -> Option<Info> {
@@ -283,10 +313,14 @@ impl SstObjectIdTrackerInner {
 #[cfg(test)]
 mod test {
 
+    use std::collections::VecDeque;
+
     use risingwave_common::try_match_expand;
 
     use crate::hummock::sstable::sstable_object_id_manager::AutoTrackerId;
-    use crate::hummock::{SstObjectIdTracker, TrackerId};
+    use crate::hummock::{
+        GetObjectId, SharedComapctorObjectIdManager, SstObjectIdTracker, TrackerId,
+    };
 
     #[tokio::test]
     async fn test_object_id_tracker_basic() {
@@ -356,4 +390,18 @@ mod test {
         object_id_tacker.remove_tracker(auto_id_3);
         assert!(object_id_tacker.tracking_object_ids().is_empty());
     }
+
+    #[tokio::test]
+    async fn test_shared_comapctor_object_id_manager() {
+        let mut pre_allocated_object_ids: VecDeque<_> = VecDeque::new();
+        pre_allocated_object_ids.extend(vec![1, 3, 5]);
+        let mut object_id_manager = SharedComapctorObjectIdManager::new(pre_allocated_object_ids);
+        assert_eq!(object_id_manager.get_new_sst_object_id().await.unwrap(), 1);
+
+        assert_eq!(object_id_manager.get_new_sst_object_id().await.unwrap(), 3);
+
+        assert_eq!(object_id_manager.get_new_sst_object_id().await.unwrap(), 5);
+
+        assert!(object_id_manager.get_new_sst_object_id().await.is_err());
+    }
 }
diff --git a/src/storage/src/hummock/state_store.rs b/src/storage/src/hummock/state_store.rs
index 853a2416ff4f6..13947802c4d9b 100644
--- a/src/storage/src/hummock/state_store.rs
+++ b/src/storage/src/hummock/state_store.rs
@@ -15,7 +15,6 @@
 use std::future::Future;
 use std::ops::Bound;
 use std::sync::atomic::Ordering as MemOrdering;
-use std::time::Duration;
 
 use bytes::Bytes;
 use itertools::Itertools;
@@ -35,6 +34,7 @@ use crate::hummock::event_handler::HummockEvent;
 use crate::hummock::store::memtable::ImmutableMemtable;
 use crate::hummock::store::state_store::LocalHummockStorage;
 use crate::hummock::store::version::read_filter_for_batch;
+use crate::hummock::utils::wait_for_epoch;
 use crate::hummock::{HummockEpoch, HummockError};
 use crate::monitor::StoreLocalStatistic;
 use crate::store::*;
@@ -186,42 +186,7 @@ impl StateStore for HummockStorage {
             }
             _ => return Ok(()),
         };
-        let mut receiver = self.version_update_notifier_tx.subscribe();
-        // avoid unnecessary check in the loop if the value does not change
-        let max_committed_epoch = *receiver.borrow_and_update();
-        if max_committed_epoch >= wait_epoch {
-            return Ok(());
-        }
-        loop {
-            match tokio::time::timeout(Duration::from_secs(30), receiver.changed()).await {
-                Err(elapsed) => {
-                    // The reason that we need to retry here is batch scan in
-                    // chain/rearrange_chain is waiting for an
-                    // uncommitted epoch carried by the CreateMV barrier, which
-                    // can take unbounded time to become committed and propagate
-                    // to the CN. We should consider removing the retry as well as wait_epoch
-                    // for chain/rearrange_chain if we enforce
-                    // chain/rearrange_chain to be scheduled on the same
-                    // CN with the same distribution as the upstream MV.
-                    // See #3845 for more details.
-                    tracing::warn!(
-                        "wait_epoch {:?} timeout when waiting for version update elapsed {:?}s",
-                        wait_epoch,
-                        elapsed
-                    );
-                    continue;
-                }
-                Ok(Err(_)) => {
-                    return Err(HummockError::wait_epoch("tx dropped").into());
-                }
-                Ok(Ok(_)) => {
-                    let max_committed_epoch = *receiver.borrow();
-                    if max_committed_epoch >= wait_epoch {
-                        return Ok(());
-                    }
-                }
-            }
-        }
+        wait_for_epoch(&self.version_update_notifier_tx, wait_epoch).await
     }
 
     async fn sync(&self, epoch: u64) -> StorageResult<SyncResult> {
diff --git a/src/storage/src/hummock/store/state_store.rs b/src/storage/src/hummock/store/state_store.rs
index 02e1f79d5a647..f5a6f65b8d6fc 100644
--- a/src/storage/src/hummock/store/state_store.rs
+++ b/src/storage/src/hummock/store/state_store.rs
@@ -38,7 +38,7 @@ use crate::hummock::shared_buffer::shared_buffer_batch::{
 use crate::hummock::store::version::{read_filter_for_local, HummockVersionReader};
 use crate::hummock::utils::{
     cmp_delete_range_left_bounds, do_delete_sanity_check, do_insert_sanity_check,
-    do_update_sanity_check, filter_with_delete_range, ENABLE_SANITY_CHECK,
+    do_update_sanity_check, filter_with_delete_range, wait_for_epoch, ENABLE_SANITY_CHECK,
 };
 use crate::hummock::write_limiter::WriteLimiterRef;
 use crate::hummock::{MemoryLimiter, SstableIterator};
@@ -84,6 +84,8 @@ pub struct LocalHummockStorage {
     stats: Arc<HummockStateStoreMetrics>,
 
     write_limiter: WriteLimiterRef,
+
+    version_update_notifier_tx: Arc<tokio::sync::watch::Sender<HummockEpoch>>,
 }
 
 impl LocalHummockStorage {
@@ -115,6 +117,10 @@ impl LocalHummockStorage {
             .await
     }
 
+    pub async fn wait_for_epoch(&self, wait_epoch: u64) -> StorageResult<()> {
+        wait_for_epoch(&self.version_update_notifier_tx, wait_epoch).await
+    }
+
     pub async fn iter_inner(
         &self,
         table_key_range: TableKeyRange,
@@ -323,12 +329,17 @@ impl LocalStateStore for LocalHummockStorage {
         self.mem_table.is_dirty()
     }
 
-    fn init(&mut self, epoch: u64) {
+    async fn init(&mut self, options: InitOptions) -> StorageResult<()> {
+        let epoch = options.epoch;
+        if self.is_replicated {
+            self.wait_for_epoch(epoch.prev).await?;
+        }
         assert!(
-            self.epoch.replace(epoch).is_none(),
+            self.epoch.replace(epoch.curr).is_none(),
             "local state store of table id {:?} is init for more than once",
             self.table_id
         );
+        Ok(())
     }
 
     fn seal_current_epoch(&mut self, next_epoch: u64) {
@@ -440,6 +451,7 @@ impl LocalHummockStorage {
         memory_limiter: Arc<MemoryLimiter>,
         write_limiter: WriteLimiterRef,
         option: NewLocalOptions,
+        version_update_notifier_tx: Arc<tokio::sync::watch::Sender<HummockEpoch>>,
     ) -> Self {
         let stats = hummock_version_reader.stats().clone();
         Self {
@@ -456,6 +468,7 @@ impl LocalHummockStorage {
             hummock_version_reader,
             stats,
             write_limiter,
+            version_update_notifier_tx,
         }
     }
 
diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs
index 5f580a22e0e83..38f7cb4e54e3f 100644
--- a/src/storage/src/hummock/store/version.rs
+++ b/src/storage/src/hummock/store/version.rs
@@ -583,7 +583,7 @@ impl HummockVersionReader {
 
         let full_key = FullKey::new(read_options.table_id, TableKey(table_key.clone()), epoch);
         for local_sst in &uncommitted_ssts {
-            local_stats.staging_imm_get_count += 1;
+            local_stats.staging_sst_get_count += 1;
             if let Some((data, data_epoch)) = get_from_sstable_info(
                 self.sstable_store.clone(),
                 local_sst,
diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs
index 983d1ebb43879..d24ff4ab09ec9 100644
--- a/src/storage/src/hummock/utils.rs
+++ b/src/storage/src/hummock/utils.rs
@@ -18,15 +18,17 @@ use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::ops::{Bound, RangeBounds};
 use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
 use std::sync::Arc;
+use std::time::Duration;
 
 use bytes::Bytes;
 use risingwave_common::cache::CachePriority;
 use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_hummock_sdk::can_concat;
 use risingwave_hummock_sdk::key::{
     bound_table_key_range, EmptySliceRef, FullKey, TableKey, UserKey,
 };
+use risingwave_hummock_sdk::{can_concat, HummockEpoch};
 use risingwave_pb::hummock::{HummockVersion, SstableInfo};
+use tokio::sync::watch::Sender;
 use tokio::sync::Notify;
 
 use super::{HummockError, HummockResult};
@@ -550,6 +552,48 @@ pub(crate) fn filter_with_delete_range<'a>(
     })
 }
 
+pub(crate) async fn wait_for_epoch(
+    notifier: &Sender<HummockEpoch>,
+    wait_epoch: u64,
+) -> StorageResult<()> {
+    let mut receiver = notifier.subscribe();
+    // avoid unnecessary check in the loop if the value does not change
+    let max_committed_epoch = *receiver.borrow_and_update();
+    if max_committed_epoch >= wait_epoch {
+        return Ok(());
+    }
+    loop {
+        match tokio::time::timeout(Duration::from_secs(30), receiver.changed()).await {
+            Err(elapsed) => {
+                // The reason that we need to retry here is batch scan in
+                // chain/rearrange_chain is waiting for an
+                // uncommitted epoch carried by the CreateMV barrier, which
+                // can take unbounded time to become committed and propagate
+                // to the CN. We should consider removing the retry as well as wait_epoch
+                // for chain/rearrange_chain if we enforce
+                // chain/rearrange_chain to be scheduled on the same
+                // CN with the same distribution as the upstream MV.
+                // See #3845 for more details.
+                tracing::warn!(
+                    "wait_epoch {:?} timeout when waiting for version update elapsed {:?}s",
+                    wait_epoch,
+                    elapsed
+                );
+                continue;
+            }
+            Ok(Err(_)) => {
+                return Err(HummockError::wait_epoch("tx dropped").into());
+            }
+            Ok(Ok(_)) => {
+                let max_committed_epoch = *receiver.borrow();
+                if max_committed_epoch >= wait_epoch {
+                    return Ok(());
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::future::{poll_fn, Future};
diff --git a/src/storage/src/hummock/vacuum.rs b/src/storage/src/hummock/vacuum.rs
index 0fa6475f7dd49..a4660e36ffd5a 100644
--- a/src/storage/src/hummock/vacuum.rs
+++ b/src/storage/src/hummock/vacuum.rs
@@ -55,49 +55,6 @@ impl Vacuum {
         true
     }
 
-    /// Wrapper method that warns on any error and doesn't propagate it.
-    /// Returns false if any error.
-    pub async fn full_scan(
-        full_scan_task: FullScanTask,
-        sstable_store: SstableStoreRef,
-        hummock_meta_client: Arc<dyn HummockMetaClient>,
-    ) -> bool {
-        tracing::info!(
-            "Try to full scan SSTs with timestamp {}",
-            full_scan_task.sst_retention_time_sec
-        );
-        let metadata_iter = match sstable_store.list_object_metadata_from_object_store().await {
-            Ok(metadata_iter) => metadata_iter,
-            Err(e) => {
-                tracing::warn!("Failed to init object iter: {:#?}", e);
-                return false;
-            }
-        };
-        let (filtered_object_ids, unfiltered_count, unfiltered_size) =
-            match Vacuum::full_scan_inner(full_scan_task, metadata_iter).await {
-                Ok(res) => res,
-                Err(e) => {
-                    tracing::warn!("Failed to iter object: {:#?}", e);
-                    return false;
-                }
-            };
-
-        match hummock_meta_client
-            .report_full_scan_task(filtered_object_ids, unfiltered_count, unfiltered_size)
-            .await
-        {
-            Ok(_) => {
-                tracing::info!("Finished full scan SSTs");
-            }
-            Err(e) => {
-                tracing::warn!("Failed to report full scan task: {:#?}", e);
-                return false;
-            }
-        }
-        true
-    }
-
-    /// Returns **filtered** object ids, and **unfiltered** total object count and size.
     pub async fn full_scan_inner(
         full_scan_task: FullScanTask,
         metadata_iter: ObjectMetadataIter,
@@ -134,4 +91,47 @@ impl Vacuum {
             total_object_size as u64,
         ))
     }
+
+    /// Returns **filtered** object ids, and **unfiltered** total object count and size.
+    pub async fn handle_full_scan_task(
+        full_scan_task: FullScanTask,
+        sstable_store: SstableStoreRef,
+    ) -> HummockResult<(Vec<HummockSstableObjectId>, u64, u64)> {
+        tracing::info!(
+            "Try to full scan SSTs with timestamp {}",
+            full_scan_task.sst_retention_time_sec
+        );
+        let metadata_iter = sstable_store
+            .list_object_metadata_from_object_store()
+            .await?;
+        Vacuum::full_scan_inner(full_scan_task, metadata_iter).await
+    }
+
+    pub async fn report_full_scan_task(
+        filtered_object_ids: Vec<u64>,
+        unfiltered_count: u64,
+        unfiltered_size: u64,
+        hummock_meta_client: Arc<dyn HummockMetaClient>,
+    ) -> bool {
+        tracing::info!("Try to report full scan task",);
+        tracing::info!(
+            "filtered_object_ids length =  {}, unfiltered_count = {}, unfiltered_size = {}",
+            filtered_object_ids.len(),
+            unfiltered_count,
+            unfiltered_size
+        );
+        match hummock_meta_client
+            .report_full_scan_task(filtered_object_ids, unfiltered_count, unfiltered_size)
+            .await
+        {
+            Ok(_) => {
+                tracing::info!("Finished full scan SSTs");
+            }
+            Err(e) => {
+                tracing::warn!("Failed to report full scan task: {:#?}", e);
+                return false;
+            }
+        }
+        true
+    }
 }
diff --git a/src/storage/src/mem_table.rs b/src/storage/src/mem_table.rs
index cd9eadd42db1c..1ff090690aa4a 100644
--- a/src/storage/src/mem_table.rs
+++ b/src/storage/src/mem_table.rs
@@ -539,12 +539,14 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
         self.mem_table.is_dirty()
     }
 
-    fn init(&mut self, epoch: u64) {
+    #[allow(clippy::unused_async)]
+    async fn init(&mut self, options: InitOptions) -> StorageResult<()> {
         assert!(
-            self.epoch.replace(epoch).is_none(),
+            self.epoch.replace(options.epoch.curr).is_none(),
             "local state store of table id {:?} is init for more than once",
             self.table_id
         );
+        Ok(())
     }
 
     fn seal_current_epoch(&mut self, next_epoch: u64) {
diff --git a/src/storage/src/monitor/compactor_metrics.rs b/src/storage/src/monitor/compactor_metrics.rs
index 441b5c0c78852..31a36116122a7 100644
--- a/src/storage/src/monitor/compactor_metrics.rs
+++ b/src/storage/src/monitor/compactor_metrics.rs
@@ -16,10 +16,10 @@ use std::sync::LazyLock;
 
 use prometheus::core::{AtomicU64, GenericCounter, GenericCounterVec};
 use prometheus::{
-    exponential_buckets, histogram_opts, register_counter_vec_with_registry,
-    register_histogram_vec_with_registry, register_histogram_with_registry,
-    register_int_counter_vec_with_registry, register_int_counter_with_registry,
-    register_int_gauge_with_registry, CounterVec, Histogram, HistogramVec, IntGauge, Registry,
+    exponential_buckets, histogram_opts, register_histogram_vec_with_registry,
+    register_histogram_with_registry, register_int_counter_vec_with_registry,
+    register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, HistogramVec,
+    IntGauge, Registry,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 
@@ -46,9 +46,6 @@ pub struct CompactorMetrics {
     pub iter_scan_key_counts: GenericCounterVec<AtomicU64>,
     pub write_build_l0_bytes: GenericCounter<AtomicU64>,
     pub sstable_distinct_epoch_count: Histogram,
-    pub preload_io_count: GenericCounter<AtomicU64>,
-    pub refill_cache_duration: Histogram,
-    pub refill_data_file_cache_count: CounterVec,
     pub compaction_event_consumed_latency: Histogram,
     pub compaction_event_loop_iteration_latency: Histogram,
 }
@@ -98,19 +95,7 @@ impl CompactorMetrics {
         );
         let get_table_id_total_time_duration =
             register_histogram_with_registry!(opts, registry).unwrap();
-        let opts = histogram_opts!(
-            "compute_refill_cache_duration",
-            "compute_refill_cache_duration",
-            time_buckets.clone()
-        );
-        let refill_cache_duration = register_histogram_with_registry!(opts, registry).unwrap();
-        let refill_data_file_cache_count = register_counter_vec_with_registry!(
-            "compute_refill_data_file_cache_count",
-            "compute refill data file cache count",
-            &["extra"],
-            registry
-        )
-        .unwrap();
+
         let opts = histogram_opts!(
             "compactor_remote_read_time",
             "Total time of operations which read from remote storage when enable prefetch",
@@ -235,12 +220,6 @@ impl CompactorMetrics {
 
         let sstable_distinct_epoch_count =
             register_histogram_with_registry!(opts, registry).unwrap();
-        let preload_io_count = register_int_counter_with_registry!(
-            "sstable_preload_io_count",
-            "Total number of preload io count",
-            registry
-        )
-        .unwrap();
 
         let opts = histogram_opts!(
             "compactor_compaction_event_consumed_latency",
@@ -280,9 +259,6 @@ impl CompactorMetrics {
             iter_scan_key_counts,
             write_build_l0_bytes,
             sstable_distinct_epoch_count,
-            preload_io_count,
-            refill_cache_duration,
-            refill_data_file_cache_count,
             compaction_event_consumed_latency,
             compaction_event_loop_iteration_latency,
         }
diff --git a/src/storage/src/monitor/hummock_state_store_metrics.rs b/src/storage/src/monitor/hummock_state_store_metrics.rs
index 0606920186ab2..77043b32ab455 100644
--- a/src/storage/src/monitor/hummock_state_store_metrics.rs
+++ b/src/storage/src/monitor/hummock_state_store_metrics.rs
@@ -12,16 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::{Arc, LazyLock};
+use std::sync::{Arc, OnceLock};
 
-use prometheus::core::{
-    AtomicU64, Collector, Desc, GenericCounter, GenericCounterVec, GenericGauge,
-};
+use prometheus::core::{AtomicU64, Collector, Desc, GenericCounter, GenericGauge};
 use prometheus::{
     exponential_buckets, histogram_opts, proto, register_histogram_vec_with_registry,
-    register_int_counter_vec_with_registry, register_int_gauge_with_registry, Gauge, HistogramVec,
-    IntGauge, Opts, Registry,
+    register_int_counter_vec_with_registry, register_int_gauge_with_registry, Gauge, IntGauge,
+    Opts, Registry,
 };
+use risingwave_common::config::MetricLevel;
+use risingwave_common::metrics::{RelabeledCounterVec, RelabeledHistogramVec};
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use tracing::warn;
 
@@ -32,29 +32,29 @@ use tracing::warn;
 /// job or an executor should be collected by views like `StateStats` and `JobStats`.
 #[derive(Debug, Clone)]
 pub struct HummockStateStoreMetrics {
-    pub bloom_filter_true_negative_counts: GenericCounterVec<AtomicU64>,
-    pub bloom_filter_check_counts: GenericCounterVec<AtomicU64>,
-    pub iter_merge_sstable_counts: HistogramVec,
-    pub sst_store_block_request_counts: GenericCounterVec<AtomicU64>,
-    pub iter_scan_key_counts: GenericCounterVec<AtomicU64>,
-    pub get_shared_buffer_hit_counts: GenericCounterVec<AtomicU64>,
-    pub remote_read_time: HistogramVec,
-    pub iter_fetch_meta_duration: HistogramVec,
+    pub bloom_filter_true_negative_counts: RelabeledCounterVec,
+    pub bloom_filter_check_counts: RelabeledCounterVec,
+    pub iter_merge_sstable_counts: RelabeledHistogramVec,
+    pub sst_store_block_request_counts: RelabeledCounterVec,
+    pub iter_scan_key_counts: RelabeledCounterVec,
+    pub get_shared_buffer_hit_counts: RelabeledCounterVec,
+    pub remote_read_time: RelabeledHistogramVec,
+    pub iter_fetch_meta_duration: RelabeledHistogramVec,
     pub iter_fetch_meta_cache_unhits: IntGauge,
     pub iter_slow_fetch_meta_cache_unhits: IntGauge,
 
-    pub read_req_bloom_filter_positive_counts: GenericCounterVec<AtomicU64>,
-    pub read_req_positive_but_non_exist_counts: GenericCounterVec<AtomicU64>,
-    pub read_req_check_bloom_filter_counts: GenericCounterVec<AtomicU64>,
+    pub read_req_bloom_filter_positive_counts: RelabeledCounterVec,
+    pub read_req_positive_but_non_exist_counts: RelabeledCounterVec,
+    pub read_req_check_bloom_filter_counts: RelabeledCounterVec,
 
-    pub write_batch_tuple_counts: GenericCounterVec<AtomicU64>,
-    pub write_batch_duration: HistogramVec,
-    pub write_batch_size: HistogramVec,
+    pub write_batch_tuple_counts: RelabeledCounterVec,
+    pub write_batch_duration: RelabeledHistogramVec,
+    pub write_batch_size: RelabeledHistogramVec,
 
     // finished task counts
-    pub merge_imm_task_counts: GenericCounterVec<AtomicU64>,
+    pub merge_imm_task_counts: RelabeledCounterVec,
     // merge imm ops
-    pub merge_imm_batch_memory_sz: GenericCounterVec<AtomicU64>,
+    pub merge_imm_batch_memory_sz: RelabeledCounterVec,
 
     // spill task counts from unsealed
     pub spill_task_counts_from_unsealed: GenericCounter<AtomicU64>,
@@ -69,11 +69,16 @@ pub struct HummockStateStoreMetrics {
     pub uploader_uploading_task_size: GenericGauge<AtomicU64>,
 }
 
-pub static GLOBAL_HUMMOCK_STATE_STORE_METRICS: LazyLock<HummockStateStoreMetrics> =
-    LazyLock::new(|| HummockStateStoreMetrics::new(&GLOBAL_METRICS_REGISTRY));
+pub static GLOBAL_HUMMOCK_STATE_STORE_METRICS: OnceLock<HummockStateStoreMetrics> = OnceLock::new();
+
+pub fn global_hummock_state_store_metrics(metric_level: MetricLevel) -> HummockStateStoreMetrics {
+    GLOBAL_HUMMOCK_STATE_STORE_METRICS
+        .get_or_init(|| HummockStateStoreMetrics::new(&GLOBAL_METRICS_REGISTRY, metric_level))
+        .clone()
+}
 
 impl HummockStateStoreMetrics {
-    fn new(registry: &Registry) -> Self {
+    pub fn new(registry: &Registry, metric_level: MetricLevel) -> Self {
         // 10ms ~ max 2.7h
         let time_buckets = exponential_buckets(0.01, 10.0, 7).unwrap();
         let bloom_filter_true_negative_counts = register_int_counter_vec_with_registry!(
@@ -83,6 +88,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let bloom_filter_true_negative_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            bloom_filter_true_negative_counts,
+            metric_level,
+        );
 
         let bloom_filter_check_counts = register_int_counter_vec_with_registry!(
             "state_store_bloom_filter_check_counts",
@@ -91,6 +101,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let bloom_filter_check_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            bloom_filter_check_counts,
+            metric_level,
+        );
 
         // ----- iter -----
         let opts = histogram_opts!(
@@ -100,6 +115,11 @@ impl HummockStateStoreMetrics {
         );
         let iter_merge_sstable_counts =
             register_histogram_vec_with_registry!(opts, &["table_id", "type"], registry).unwrap();
+        let iter_merge_sstable_counts = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            iter_merge_sstable_counts,
+            metric_level,
+        );
 
         // ----- sst store -----
         let sst_store_block_request_counts = register_int_counter_vec_with_registry!(
@@ -109,6 +129,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let sst_store_block_request_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Critical,
+            sst_store_block_request_counts,
+            metric_level,
+        );
 
         let iter_scan_key_counts = register_int_counter_vec_with_registry!(
             "state_store_iter_scan_key_counts",
@@ -117,6 +142,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let iter_scan_key_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Info,
+            iter_scan_key_counts,
+            metric_level,
+        );
 
         let get_shared_buffer_hit_counts = register_int_counter_vec_with_registry!(
             "state_store_get_shared_buffer_hit_counts",
@@ -125,6 +155,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let get_shared_buffer_hit_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            get_shared_buffer_hit_counts,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_remote_read_time_per_task",
@@ -133,6 +168,11 @@ impl HummockStateStoreMetrics {
         );
         let remote_read_time =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let remote_read_time = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            remote_read_time,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_iter_fetch_meta_duration",
@@ -141,6 +181,11 @@ impl HummockStateStoreMetrics {
         );
         let iter_fetch_meta_duration =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let iter_fetch_meta_duration = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Info,
+            iter_fetch_meta_duration,
+            metric_level,
+        );
 
         let iter_fetch_meta_cache_unhits = register_int_gauge_with_registry!(
             "state_store_iter_fetch_meta_cache_unhits",
@@ -164,6 +209,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let write_batch_tuple_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            write_batch_tuple_counts,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
                 "state_store_write_batch_duration",
@@ -172,6 +222,11 @@ impl HummockStateStoreMetrics {
             );
         let write_batch_duration =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let write_batch_duration = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            write_batch_duration,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_write_batch_size",
@@ -180,6 +235,11 @@ impl HummockStateStoreMetrics {
         );
         let write_batch_size =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let write_batch_size = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            write_batch_size,
+            metric_level,
+        );
 
         let merge_imm_task_counts = register_int_counter_vec_with_registry!(
             "state_store_merge_imm_task_counts",
@@ -188,6 +248,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let merge_imm_task_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            merge_imm_task_counts,
+            metric_level,
+        );
 
         let merge_imm_batch_memory_sz = register_int_counter_vec_with_registry!(
             "state_store_merge_imm_memory_sz",
@@ -196,6 +261,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let merge_imm_batch_memory_sz = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            merge_imm_batch_memory_sz,
+            metric_level,
+        );
 
         let spill_task_counts = register_int_counter_vec_with_registry!(
             "state_store_spill_task_counts",
@@ -204,6 +274,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let spill_task_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            spill_task_counts,
+            metric_level,
+        );
 
         let spill_task_size = register_int_counter_vec_with_registry!(
             "state_store_spill_task_size",
@@ -212,6 +287,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let spill_task_size = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            spill_task_size,
+            metric_level,
+        );
 
         let uploader_uploading_task_size = GenericGauge::new(
             "state_store_uploader_uploading_task_size",
@@ -229,6 +309,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let read_req_bloom_filter_positive_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Info,
+            read_req_bloom_filter_positive_counts,
+            metric_level,
+        );
 
         let read_req_positive_but_non_exist_counts = register_int_counter_vec_with_registry!(
             "state_store_read_req_positive_but_non_exist_counts",
@@ -237,6 +322,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let read_req_positive_but_non_exist_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Info,
+            read_req_positive_but_non_exist_counts,
+            metric_level,
+        );
 
         let read_req_check_bloom_filter_counts = register_int_counter_vec_with_registry!(
             "state_store_read_req_check_bloom_filter_counts",
@@ -245,6 +335,11 @@ impl HummockStateStoreMetrics {
             registry
         )
         .unwrap();
+        let read_req_check_bloom_filter_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Info,
+            read_req_check_bloom_filter_counts,
+            metric_level,
+        );
 
         Self {
             bloom_filter_true_negative_counts,
@@ -274,7 +369,7 @@ impl HummockStateStoreMetrics {
     }
 
     pub fn unused() -> Self {
-        GLOBAL_HUMMOCK_STATE_STORE_METRICS.clone()
+        global_hummock_state_store_metrics(MetricLevel::Disabled)
     }
 }
 
diff --git a/src/storage/src/monitor/monitored_storage_metrics.rs b/src/storage/src/monitor/monitored_storage_metrics.rs
index 19235fe0cfbcc..a1517d98918ac 100644
--- a/src/storage/src/monitor/monitored_storage_metrics.rs
+++ b/src/storage/src/monitor/monitored_storage_metrics.rs
@@ -12,40 +12,45 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::LazyLock;
+use std::sync::OnceLock;
 
-use prometheus::core::{AtomicU64, GenericCounterVec};
 use prometheus::{
     exponential_buckets, histogram_opts, linear_buckets, register_histogram_vec_with_registry,
-    register_histogram_with_registry, register_int_counter_vec_with_registry, Histogram,
-    HistogramVec, Registry,
+    register_histogram_with_registry, register_int_counter_vec_with_registry, Histogram, Registry,
 };
+use risingwave_common::config::MetricLevel;
+use risingwave_common::metrics::{RelabeledCounterVec, RelabeledHistogramVec};
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 
 /// [`MonitoredStorageMetrics`] stores the performance and IO metrics of Storage.
 #[derive(Debug, Clone)]
 pub struct MonitoredStorageMetrics {
-    pub get_duration: HistogramVec,
-    pub get_key_size: HistogramVec,
-    pub get_value_size: HistogramVec,
+    pub get_duration: RelabeledHistogramVec,
+    pub get_key_size: RelabeledHistogramVec,
+    pub get_value_size: RelabeledHistogramVec,
 
-    pub iter_size: HistogramVec,
-    pub iter_item: HistogramVec,
-    pub iter_init_duration: HistogramVec,
-    pub iter_scan_duration: HistogramVec,
-    pub may_exist_duration: HistogramVec,
+    pub iter_size: RelabeledHistogramVec,
+    pub iter_item: RelabeledHistogramVec,
+    pub iter_init_duration: RelabeledHistogramVec,
+    pub iter_scan_duration: RelabeledHistogramVec,
+    pub may_exist_duration: RelabeledHistogramVec,
 
-    pub iter_in_process_counts: GenericCounterVec<AtomicU64>,
+    pub iter_in_process_counts: RelabeledCounterVec,
 
     pub sync_duration: Histogram,
     pub sync_size: Histogram,
 }
 
-pub static GLOBAL_STORAGE_METRICS: LazyLock<MonitoredStorageMetrics> =
-    LazyLock::new(|| MonitoredStorageMetrics::new(&GLOBAL_METRICS_REGISTRY));
+pub static GLOBAL_STORAGE_METRICS: OnceLock<MonitoredStorageMetrics> = OnceLock::new();
+
+pub fn global_storage_metrics(metric_level: MetricLevel) -> MonitoredStorageMetrics {
+    GLOBAL_STORAGE_METRICS
+        .get_or_init(|| MonitoredStorageMetrics::new(&GLOBAL_METRICS_REGISTRY, metric_level))
+        .clone()
+}
 
 impl MonitoredStorageMetrics {
-    fn new(registry: &Registry) -> Self {
+    pub fn new(registry: &Registry, metric_level: MetricLevel) -> Self {
         // 256B ~ max 4GB
         let size_buckets = exponential_buckets(256.0, 16.0, 7).unwrap();
         // 10ms ~ max 2.7h
@@ -58,6 +63,11 @@ impl MonitoredStorageMetrics {
         );
         let get_key_size =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let get_key_size = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            get_key_size,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_get_value_size",
@@ -66,6 +76,11 @@ impl MonitoredStorageMetrics {
         );
         let get_value_size =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let get_value_size = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            get_value_size,
+            metric_level,
+        );
 
         let mut buckets = exponential_buckets(0.000004, 2.0, 4).unwrap(); // 4 ~ 32us
         buckets.extend(linear_buckets(0.00006, 0.00004, 5).unwrap()); // 60 ~ 220us.
@@ -81,6 +96,11 @@ impl MonitoredStorageMetrics {
         let get_duration =
             register_histogram_vec_with_registry!(get_duration_opts, &["table_id"], registry)
                 .unwrap();
+        let get_duration = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Critical,
+            get_duration,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_iter_size",
@@ -89,6 +109,8 @@ impl MonitoredStorageMetrics {
         );
         let iter_size =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let iter_size =
+            RelabeledHistogramVec::with_metric_level(MetricLevel::Debug, iter_size, metric_level);
 
         let opts = histogram_opts!(
             "state_store_iter_item",
@@ -97,6 +119,8 @@ impl MonitoredStorageMetrics {
         );
         let iter_item =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let iter_item =
+            RelabeledHistogramVec::with_metric_level(MetricLevel::Debug, iter_item, metric_level);
 
         let opts = histogram_opts!(
             "state_store_iter_init_duration",
@@ -105,6 +129,11 @@ impl MonitoredStorageMetrics {
         );
         let iter_init_duration =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let iter_init_duration = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Critical,
+            iter_init_duration,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_iter_scan_duration",
@@ -113,6 +142,11 @@ impl MonitoredStorageMetrics {
         );
         let iter_scan_duration =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let iter_scan_duration = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Critical,
+            iter_scan_duration,
+            metric_level,
+        );
 
         let iter_in_process_counts = register_int_counter_vec_with_registry!(
             "state_store_iter_in_process_counts",
@@ -121,6 +155,11 @@ impl MonitoredStorageMetrics {
             registry
         )
         .unwrap();
+        let iter_in_process_counts = RelabeledCounterVec::with_metric_level(
+            MetricLevel::Debug,
+            iter_in_process_counts,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_may_exist_duration",
@@ -129,6 +168,11 @@ impl MonitoredStorageMetrics {
         );
         let may_exist_duration =
             register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let may_exist_duration = RelabeledHistogramVec::with_metric_level(
+            MetricLevel::Debug,
+            may_exist_duration,
+            metric_level,
+        );
 
         let opts = histogram_opts!(
             "state_store_sync_duration",
@@ -160,6 +204,6 @@ impl MonitoredStorageMetrics {
     }
 
     pub fn unused() -> Self {
-        GLOBAL_STORAGE_METRICS.clone()
+        global_storage_metrics(MetricLevel::Disabled)
     }
 }
diff --git a/src/storage/src/monitor/monitored_store.rs b/src/storage/src/monitor/monitored_store.rs
index b8ed350c1fd85..77924a999709d 100644
--- a/src/storage/src/monitor/monitored_store.rs
+++ b/src/storage/src/monitor/monitored_store.rs
@@ -271,9 +271,8 @@ impl<S: LocalStateStore> LocalStateStore for MonitoredStateStore<S> {
         self.inner.is_dirty()
     }
 
-    fn init(&mut self, epoch: u64) {
-        // TODO: may collect metrics
-        self.inner.init(epoch)
+    async fn init(&mut self, options: InitOptions) -> StorageResult<()> {
+        self.inner.init(options).await
     }
 
     fn seal_current_epoch(&mut self, next_epoch: u64) {
diff --git a/src/storage/src/monitor/traced_store.rs b/src/storage/src/monitor/traced_store.rs
index f7727c5d0e9f5..efa4b1111dcb5 100644
--- a/src/storage/src/monitor/traced_store.rs
+++ b/src/storage/src/monitor/traced_store.rs
@@ -195,9 +195,10 @@ impl<S: LocalStateStore> LocalStateStore for TracedStateStore<S> {
         res
     }
 
-    fn init(&mut self, epoch: u64) {
-        let _span = TraceSpan::new_local_storage_init_span(epoch, self.storage_type);
-        self.inner.init(epoch)
+    async fn init(&mut self, options: InitOptions) -> StorageResult<()> {
+        let _span =
+            TraceSpan::new_local_storage_init_span(options.clone().into(), self.storage_type);
+        self.inner.init(options).await
     }
 
     fn seal_current_epoch(&mut self, next_epoch: u64) {
diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs
index a6f7101a51f0c..22dac37c3a3f0 100644
--- a/src/storage/src/opts.rs
+++ b/src/storage/src/opts.rs
@@ -79,7 +79,9 @@ pub struct StorageOpts {
     pub data_file_cache_flush_rate_limit_mb: usize,
     pub data_file_cache_reclaim_rate_limit_mb: usize,
 
-    pub data_file_cache_refill_levels: Vec<u32>,
+    pub cache_refill_data_refill_levels: Vec<u32>,
+    pub cache_refill_timeout_ms: u64,
+    pub cache_refill_concurrency: usize,
 
     pub meta_file_cache_dir: String,
     pub meta_file_cache_capacity_mb: usize,
@@ -172,8 +174,6 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             data_file_cache_rated_random_rate_mb: c.storage.data_file_cache.rated_random_rate_mb,
             data_file_cache_flush_rate_limit_mb: c.storage.data_file_cache.flush_rate_limit_mb,
             data_file_cache_reclaim_rate_limit_mb: c.storage.data_file_cache.reclaim_rate_limit_mb,
-            data_file_cache_refill_levels: c.storage.data_file_cache.refill_levels.clone(),
-
             meta_file_cache_dir: c.storage.meta_file_cache.dir.clone(),
             meta_file_cache_capacity_mb: c.storage.meta_file_cache.capacity_mb,
             meta_file_cache_file_capacity_mb: c.storage.meta_file_cache.file_capacity_mb,
@@ -194,6 +194,9 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             meta_file_cache_rated_random_rate_mb: c.storage.meta_file_cache.rated_random_rate_mb,
             meta_file_cache_flush_rate_limit_mb: c.storage.meta_file_cache.flush_rate_limit_mb,
             meta_file_cache_reclaim_rate_limit_mb: c.storage.meta_file_cache.reclaim_rate_limit_mb,
+            cache_refill_data_refill_levels: c.storage.cache_refill.data_refill_levels.clone(),
+            cache_refill_timeout_ms: c.storage.cache_refill.timeout_ms,
+            cache_refill_concurrency: c.storage.cache_refill.concurrency,
             max_preload_wait_time_mill: c.storage.max_preload_wait_time_mill,
             object_store_streaming_read_timeout_ms: c
                 .storage
diff --git a/src/storage/src/panic_store.rs b/src/storage/src/panic_store.rs
index e77de344faa6a..53162e29637fd 100644
--- a/src/storage/src/panic_store.rs
+++ b/src/storage/src/panic_store.rs
@@ -120,7 +120,8 @@ impl LocalStateStore for PanicStateStore {
         panic!("should not operate on the panic state store!");
     }
 
-    fn init(&mut self, _epoch: u64) {
+    #[allow(clippy::unused_async)]
+    async fn init(&mut self, _epoch: InitOptions) -> StorageResult<()> {
         panic!("should not operate on the panic state store!");
     }
 
diff --git a/src/storage/src/row_serde/row_serde_util.rs b/src/storage/src/row_serde/row_serde_util.rs
index e85f69eb050ce..d426789829797 100644
--- a/src/storage/src/row_serde/row_serde_util.rs
+++ b/src/storage/src/row_serde/row_serde_util.rs
@@ -42,9 +42,3 @@ pub fn deserialize_pk_with_vnode(
     let pk = deserializer.deserialize(&key[VirtualNode::SIZE..])?;
     Ok((vnode, pk))
 }
-
-pub fn parse_raw_key_to_vnode_and_key(raw_key: &[u8]) -> (VirtualNode, &[u8]) {
-    let (vnode_bytes, key_bytes) = raw_key.split_at(VirtualNode::SIZE);
-    let vnode = VirtualNode::from_be_bytes(vnode_bytes.try_into().unwrap());
-    (vnode, key_bytes)
-}
diff --git a/src/storage/src/store.rs b/src/storage/src/store.rs
index c17b3175714c3..4150e31d616e8 100644
--- a/src/storage/src/store.rs
+++ b/src/storage/src/store.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::default::Default;
 use std::future::Future;
 use std::ops::Bound;
 use std::sync::Arc;
@@ -20,11 +21,12 @@ use bytes::Bytes;
 use futures::{Stream, StreamExt, TryStreamExt};
 use futures_async_stream::try_stream;
 use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_common::util::epoch::Epoch;
+use risingwave_common::util::epoch::{Epoch, EpochPair};
 use risingwave_hummock_sdk::key::{FullKey, KeyPayloadType};
 use risingwave_hummock_sdk::{HummockReadEpoch, LocalSstableInfo};
 use risingwave_hummock_trace::{
-    TracedNewLocalOptions, TracedPrefetchOptions, TracedReadOptions, TracedWriteOptions,
+    TracedInitOptions, TracedNewLocalOptions, TracedPrefetchOptions, TracedReadOptions,
+    TracedWriteOptions,
 };
 
 use crate::error::{StorageError, StorageResult};
@@ -242,7 +244,13 @@ pub trait LocalStateStore: StaticSendSync {
 
     fn is_dirty(&self) -> bool;
 
-    fn init(&mut self, epoch: u64);
+    /// Initializes the state store with given `epoch` pair.
+    /// Typically we will use `epoch.curr` as the initialized epoch,
+    /// Since state table will begin as empty.
+    /// In some cases like replicated state table, state table may not be empty initially,
+    /// as such we need to wait for `epoch.prev` checkpoint to complete,
+    /// hence this interface is made async.
+    fn init(&mut self, epoch: InitOptions) -> impl Future<Output = StorageResult<()>> + Send + '_;
 
     /// Updates the monotonically increasing write epoch to `new_epoch`.
     /// All writes after this function is called will be tagged with `new_epoch`. In other words,
@@ -276,7 +284,11 @@ pub struct PrefetchOptions {
 
 impl PrefetchOptions {
     pub fn new_for_exhaust_iter() -> Self {
-        Self { exhaust_iter: true }
+        Self::new_with_exhaust_iter(true)
+    }
+
+    pub fn new_with_exhaust_iter(exhaust_iter: bool) -> Self {
+        Self { exhaust_iter }
     }
 }
 
@@ -442,3 +454,36 @@ impl NewLocalOptions {
         }
     }
 }
+
+#[derive(Clone)]
+pub struct InitOptions {
+    pub epoch: EpochPair,
+}
+
+impl InitOptions {
+    pub fn new_with_epoch(epoch: EpochPair) -> Self {
+        Self { epoch }
+    }
+}
+
+impl From<EpochPair> for InitOptions {
+    fn from(value: EpochPair) -> Self {
+        Self { epoch: value }
+    }
+}
+
+impl From<InitOptions> for TracedInitOptions {
+    fn from(value: InitOptions) -> Self {
+        TracedInitOptions {
+            epoch: value.epoch.into(),
+        }
+    }
+}
+
+impl From<TracedInitOptions> for InitOptions {
+    fn from(value: TracedInitOptions) -> Self {
+        InitOptions {
+            epoch: value.epoch.into(),
+        }
+    }
+}
diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs
index 1655d774e2ee8..54d79c11d741f 100644
--- a/src/storage/src/store_impl.rs
+++ b/src/storage/src/store_impl.rs
@@ -23,14 +23,14 @@ use risingwave_object_store::object::parse_remote_object_store;
 
 use crate::error::StorageResult;
 use crate::filter_key_extractor::{
-    FilterKeyExtractorManager, FilterKeyExtractorManagerRef, RemoteTableAccessor,
+    FilterKeyExtractorManager, RemoteTableAccessor, RpcFilterKeyExtractorManager,
 };
 use crate::hummock::backup_reader::BackupReaderRef;
 use crate::hummock::hummock_meta_client::MonitoredHummockMetaClient;
 use crate::hummock::sstable_store::SstableStoreRef;
 use crate::hummock::{
-    FileCache, FoyerRuntimeConfig, FoyerStoreConfig, HummockError, HummockStorage, MemoryLimiter,
-    SstableObjectIdManagerRef, SstableStore,
+    set_foyer_metrics_registry, FileCache, FoyerRuntimeConfig, FoyerStoreConfig, HummockError,
+    HummockStorage, MemoryLimiter, SstableObjectIdManagerRef, SstableStore,
 };
 use crate::memory::sled::SledStateStore;
 use crate::memory::MemoryStateStore;
@@ -447,11 +447,12 @@ pub mod verify {
             self.actual.flush(delete_ranges).await
         }
 
-        fn init(&mut self, epoch: u64) {
-            self.actual.init(epoch);
+        async fn init(&mut self, options: InitOptions) -> StorageResult<()> {
+            self.actual.init(options.clone()).await?;
             if let Some(expected) = &mut self.expected {
-                expected.init(epoch);
+                expected.init(options).await?;
             }
+            Ok(())
         }
 
         fn seal_current_epoch(&mut self, next_epoch: u64) {
@@ -541,12 +542,15 @@ impl StateStoreImpl {
         storage_metrics: Arc<MonitoredStorageMetrics>,
         compactor_metrics: Arc<CompactorMetrics>,
     ) -> StorageResult<Self> {
+        set_foyer_metrics_registry(GLOBAL_METRICS_REGISTRY.clone());
+
         let data_file_cache = if opts.data_file_cache_dir.is_empty() {
             FileCache::none()
         } else {
             const MB: usize = 1024 * 1024;
 
             let foyer_store_config = FoyerStoreConfig {
+                name: "data".to_string(),
                 dir: PathBuf::from(opts.data_file_cache_dir.clone()),
                 capacity: opts.data_file_cache_capacity_mb * MB,
                 file_capacity: opts.data_file_cache_file_capacity_mb * MB,
@@ -562,9 +566,7 @@ impl StateStoreImpl {
                 reclaim_rate_limit: opts.data_file_cache_reclaim_rate_limit_mb * MB,
                 recover_concurrency: opts.data_file_cache_recover_concurrency,
                 event_listener: vec![],
-                prometheus_registry: Some(GLOBAL_METRICS_REGISTRY.clone()),
-                prometheus_namespace: Some("data".to_string()),
-                enable_filter: !opts.data_file_cache_refill_levels.is_empty(),
+                enable_filter: !opts.cache_refill_data_refill_levels.is_empty(),
             };
             let config = FoyerRuntimeConfig {
                 foyer_store_config,
@@ -581,6 +583,7 @@ impl StateStoreImpl {
             const MB: usize = 1024 * 1024;
 
             let foyer_store_config = FoyerStoreConfig {
+                name: "meta".to_string(),
                 dir: PathBuf::from(opts.meta_file_cache_dir.clone()),
                 capacity: opts.meta_file_cache_capacity_mb * MB,
                 file_capacity: opts.meta_file_cache_file_capacity_mb * MB,
@@ -596,8 +599,6 @@ impl StateStoreImpl {
                 reclaim_rate_limit: opts.meta_file_cache_reclaim_rate_limit_mb * MB,
                 recover_concurrency: opts.meta_file_cache_recover_concurrency,
                 event_listener: vec![],
-                prometheus_registry: Some(GLOBAL_METRICS_REGISTRY.clone()),
-                prometheus_namespace: Some("meta".to_string()),
                 enable_filter: false,
             };
             let config = FoyerRuntimeConfig {
@@ -635,7 +636,7 @@ impl StateStoreImpl {
                 ));
                 let notification_client =
                     RpcNotificationClient::new(hummock_meta_client.get_inner().clone());
-                let key_filter_manager = Arc::new(FilterKeyExtractorManager::new(Box::new(
+                let key_filter_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new(
                     RemoteTableAccessor::new(hummock_meta_client.get_inner().clone()),
                 )));
                 let inner = HummockStorage::new(
@@ -674,7 +675,7 @@ impl StateStoreImpl {
 pub trait HummockTrait {
     fn sstable_object_id_manager(&self) -> &SstableObjectIdManagerRef;
     fn sstable_store(&self) -> SstableStoreRef;
-    fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManagerRef;
+    fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager;
     fn get_memory_limiter(&self) -> Arc<MemoryLimiter>;
     fn backup_reader(&self) -> BackupReaderRef;
     fn as_hummock(&self) -> Option<&HummockStorage>;
@@ -689,7 +690,7 @@ impl HummockTrait for HummockStorage {
         self.sstable_store()
     }
 
-    fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManagerRef {
+    fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager {
         self.filter_key_extractor_manager()
     }
 
@@ -821,7 +822,7 @@ pub mod boxed_state_store {
 
         fn is_dirty(&self) -> bool;
 
-        fn init(&mut self, epoch: u64);
+        async fn init(&mut self, epoch: InitOptions) -> StorageResult<()>;
 
         fn seal_current_epoch(&mut self, next_epoch: u64);
     }
@@ -876,8 +877,8 @@ pub mod boxed_state_store {
             self.is_dirty()
         }
 
-        fn init(&mut self, epoch: u64) {
-            self.init(epoch)
+        async fn init(&mut self, options: InitOptions) -> StorageResult<()> {
+            self.init(options).await
         }
 
         fn seal_current_epoch(&mut self, next_epoch: u64) {
@@ -942,8 +943,11 @@ pub mod boxed_state_store {
             self.deref().is_dirty()
         }
 
-        fn init(&mut self, epoch: u64) {
-            self.deref_mut().init(epoch)
+        fn init(
+            &mut self,
+            options: InitOptions,
+        ) -> impl Future<Output = StorageResult<()>> + Send + '_ {
+            self.deref_mut().init(options)
         }
 
         fn seal_current_epoch(&mut self, next_epoch: u64) {
diff --git a/src/storage/src/table/batch_table/storage_table.rs b/src/storage/src/table/batch_table/storage_table.rs
index 80366748a9858..dc386a719ec22 100644
--- a/src/storage/src/table/batch_table/storage_table.rs
+++ b/src/storage/src/table/batch_table/storage_table.rs
@@ -39,14 +39,12 @@ use tracing::trace;
 
 use crate::error::{StorageError, StorageResult};
 use crate::hummock::CachePolicy;
-use crate::row_serde::row_serde_util::{
-    parse_raw_key_to_vnode_and_key, serialize_pk, serialize_pk_with_vnode,
-};
+use crate::row_serde::row_serde_util::{serialize_pk, serialize_pk_with_vnode};
 use crate::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew};
 use crate::row_serde::{find_columns_by_ids, ColumnMapping};
 use crate::store::{PrefetchOptions, ReadOptions};
 use crate::table::merge_sort::merge_sort;
-use crate::table::{compute_vnode, Distribution, TableIter, DEFAULT_VNODE};
+use crate::table::{compute_vnode, Distribution, KeyedRow, TableIter, DEFAULT_VNODE};
 use crate::StateStore;
 
 /// [`StorageTableInner`] is the interface accessing relational data in KV(`StateStore`) with
@@ -285,7 +283,6 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         self.table_id
     }
 }
-
 /// Point get
 impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
     /// Get vnode value with given primary key.
@@ -380,7 +377,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
     }
 }
 
-pub trait PkAndRowStream = Stream<Item = StorageResult<(Vec<u8>, OwnedRow)>> + Send;
+pub trait PkAndRowStream = Stream<Item = StorageResult<KeyedRow<Bytes>>> + Send;
 
 /// The row iterator of the storage table.
 /// The wrapper of [`StorageTableInnerIter`] if pk is not persisted.
@@ -392,7 +389,7 @@ impl<S: PkAndRowStream + Unpin> TableIter for S {
         self.next()
             .await
             .transpose()
-            .map(|r| r.map(|(_pk, row)| row))
+            .map(|r| r.map(|keyed_row| keyed_row.into_owned_row()))
     }
 }
 
@@ -709,20 +706,18 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterInner<S, SD> {
     }
 
     /// Yield a row with its primary key.
-    #[try_stream(ok = (Vec<u8>, OwnedRow), error = StorageError)]
+    #[try_stream(ok = KeyedRow<Bytes>, error = StorageError)]
     async fn into_stream(self) {
         use futures::TryStreamExt;
 
         // No need for table id and epoch.
-        let iter = self.iter.map_ok(|(k, v)| (k.user_key.table_key.0, v));
+        let iter = self.iter.map_ok(|(k, v)| (k.user_key.table_key, v));
         futures::pin_mut!(iter);
-        while let Some((raw_key, value)) = iter
+        while let Some((table_key, value)) = iter
             .try_next()
             .verbose_instrument_await("storage_table_iter_next")
             .await?
         {
-            let (_, key) = parse_raw_key_to_vnode_and_key(&raw_key);
-
             let full_row = self.row_deserializer.deserialize(&value)?;
             let result_row_in_value = self
                 .mapping
@@ -732,7 +727,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterInner<S, SD> {
                 Some(key_output_indices) => {
                     let result_row_in_key = match self.pk_serializer.clone() {
                         Some(pk_serializer) => {
-                            let pk = pk_serializer.deserialize(key)?;
+                            let pk = pk_serializer.deserialize(table_key.key_part().as_ref())?;
 
                             pk.project(&self.output_row_in_key_indices).into_owned_row()
                         }
@@ -761,9 +756,17 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterInner<S, SD> {
                     }
                     let row = OwnedRow::new(result_row_vec);
 
-                    yield (key.to_vec(), row)
+                    yield KeyedRow {
+                        vnode_prefixed_key: table_key,
+                        row,
+                    }
+                }
+                None => {
+                    yield KeyedRow {
+                        vnode_prefixed_key: table_key,
+                        row: result_row_in_value,
+                    }
                 }
-                None => yield (key.to_vec(), result_row_in_value),
             }
         }
     }
diff --git a/src/storage/src/table/merge_sort.rs b/src/storage/src/table/merge_sort.rs
index b349bed10b1da..1b0ade0211029 100644
--- a/src/storage/src/table/merge_sort.rs
+++ b/src/storage/src/table/merge_sort.rs
@@ -18,47 +18,46 @@ use std::error::Error;
 
 use futures::{Stream, StreamExt};
 use futures_async_stream::try_stream;
-use risingwave_common::row::OwnedRow;
 
-pub trait MergeSortKey = Eq + PartialEq + Ord + PartialOrd;
+use super::KeyedRow;
 
-struct Node<K: MergeSortKey, S> {
+struct Node<K: AsRef<[u8]>, S> {
     stream: S,
 
     /// The next item polled from `stream` previously. Since the `eq` and `cmp` must be synchronous
     /// functions, we need to implement peeking manually.
-    peeked: (K, OwnedRow),
+    peeked: KeyedRow<K>,
 }
 
-impl<K: MergeSortKey, S> PartialEq for Node<K, S> {
+impl<K: AsRef<[u8]>, S> PartialEq for Node<K, S> {
     fn eq(&self, other: &Self) -> bool {
-        match self.peeked.0 == other.peeked.0 {
+        match self.peeked.key() == other.peeked.key() {
             true => unreachable!("primary key from different iters should be unique"),
             false => false,
         }
     }
 }
-impl<K: MergeSortKey, S> Eq for Node<K, S> {}
+impl<K: AsRef<[u8]>, S> Eq for Node<K, S> {}
 
-impl<K: MergeSortKey, S> PartialOrd for Node<K, S> {
+impl<K: AsRef<[u8]>, S> PartialOrd for Node<K, S> {
     fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<K: MergeSortKey, S> Ord for Node<K, S> {
+impl<K: AsRef<[u8]>, S> Ord for Node<K, S> {
     fn cmp(&self, other: &Self) -> std::cmp::Ordering {
         // The heap is a max heap, so we need to reverse the order.
-        self.peeked.0.cmp(&other.peeked.0).reverse()
+        self.peeked.key().cmp(other.peeked.key()).reverse()
     }
 }
 
-#[try_stream(ok=(K, OwnedRow), error=E)]
+#[try_stream(ok=KeyedRow<K>, error=E)]
 pub async fn merge_sort<'a, K, E, R>(streams: Vec<R>)
 where
-    K: MergeSortKey + 'a,
+    K: AsRef<[u8]> + 'a,
     E: Error + 'a,
-    R: Stream<Item = Result<(K, OwnedRow), E>> + 'a + Unpin,
+    R: Stream<Item = Result<KeyedRow<K>, E>> + 'a + Unpin,
 {
     let mut heap = BinaryHeap::new();
     for mut stream in streams {
@@ -80,14 +79,19 @@ where
 #[cfg(test)]
 mod tests {
     use futures_async_stream::for_await;
+    use risingwave_common::hash::VirtualNode;
+    use risingwave_common::row::OwnedRow;
     use risingwave_common::types::ScalarImpl;
+    use risingwave_hummock_sdk::key::TableKey;
 
     use super::*;
     use crate::error::StorageResult;
 
-    fn gen_pk_and_row(i: u8) -> StorageResult<(Vec<u8>, OwnedRow)> {
-        Ok((
-            vec![i],
+    fn gen_pk_and_row(i: u8) -> StorageResult<KeyedRow<Vec<u8>>> {
+        let mut key = VirtualNode::ZERO.to_be_bytes().to_vec();
+        key.extend(vec![i]);
+        Ok(KeyedRow::new(
+            TableKey(key),
             OwnedRow::new(vec![Some(ScalarImpl::Int64(i as _))]),
         ))
     }
@@ -119,7 +123,10 @@ mod tests {
 
         #[for_await]
         for (i, result) in merge_sorted.enumerate() {
-            assert_eq!(result.unwrap(), gen_pk_and_row(i as u8).unwrap());
+            let expected = gen_pk_and_row(i as u8).unwrap();
+            let actual = result.unwrap();
+            assert_eq!(actual.key(), expected.key());
+            assert_eq!(actual.into_owned_row(), expected.into_owned_row());
         }
     }
 }
diff --git a/src/storage/src/table/mod.rs b/src/storage/src/table/mod.rs
index 8535b3b2ed0af..aa876e2c6b88c 100644
--- a/src/storage/src/table/mod.rs
+++ b/src/storage/src/table/mod.rs
@@ -15,8 +15,10 @@
 pub mod batch_table;
 pub mod merge_sort;
 
+use std::ops::Deref;
 use std::sync::{Arc, LazyLock};
 
+use bytes::Bytes;
 use futures::{Stream, StreamExt};
 use itertools::Itertools;
 use risingwave_common::array::DataChunk;
@@ -26,6 +28,7 @@ use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::{OwnedRow, Row};
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::iter_util::ZipEqFast;
+use risingwave_hummock_sdk::key::TableKey;
 
 use crate::error::StorageResult;
 
@@ -92,7 +95,7 @@ pub async fn collect_data_chunk<E, S>(
     chunk_size: Option<usize>,
 ) -> Result<Option<DataChunk>, E>
 where
-    S: Stream<Item = Result<OwnedRow, E>> + Unpin,
+    S: Stream<Item = Result<KeyedRow<Bytes>, E>> + Unpin,
 {
     let mut builders = schema.create_array_builders(chunk_size.unwrap_or(0));
     let mut row_count = 0;
@@ -127,28 +130,22 @@ where
 /// Collects data chunks from stream of rows.
 pub async fn collect_data_chunk_with_builder<E, S>(
     stream: &mut S,
-    chunk_size: Option<usize>,
     builder: &mut DataChunkBuilder,
 ) -> Result<Option<DataChunk>, E>
 where
     S: Stream<Item = Result<OwnedRow, E>> + Unpin,
 {
-    for _ in 0..chunk_size.unwrap_or(usize::MAX) {
-        match stream.next().await.transpose()? {
-            Some(row) => {
-                builder.append_one_row_no_finish(row);
-            }
-            None => break,
+    // TODO(kwannoel): If necessary, we can optimize it in the future.
+    // This can be done by moving the check if builder is full from `append_one_row` to here,
+    while let Some(row) = stream.next().await.transpose()? {
+        let result = builder.append_one_row(row);
+        if let Some(chunk) = result {
+            return Ok(Some(chunk));
         }
     }
 
-    let chunk = builder.build_data_chunk();
-
-    if chunk.cardinality() == 0 {
-        Ok(None)
-    } else {
-        Ok(Some(chunk))
-    }
+    let chunk = builder.consume_all();
+    Ok(chunk)
 }
 
 pub fn get_second<T, U, E>(arg: Result<(T, U), E>) -> Result<U, E> {
@@ -208,3 +205,37 @@ fn check_vnode_is_set(vnode: VirtualNode, vnodes: &Bitmap) {
         vnode
     );
 }
+
+pub struct KeyedRow<T: AsRef<[u8]>> {
+    vnode_prefixed_key: TableKey<T>,
+    row: OwnedRow,
+}
+
+impl<T: AsRef<[u8]>> KeyedRow<T> {
+    pub fn new(table_key: TableKey<T>, row: OwnedRow) -> Self {
+        Self {
+            vnode_prefixed_key: table_key,
+            row,
+        }
+    }
+
+    pub fn into_owned_row(self) -> OwnedRow {
+        self.row
+    }
+
+    pub fn vnode(&self) -> VirtualNode {
+        self.vnode_prefixed_key.vnode_part()
+    }
+
+    pub fn key(&self) -> &[u8] {
+        self.vnode_prefixed_key.key_part()
+    }
+}
+
+impl<T: AsRef<[u8]>> Deref for KeyedRow<T> {
+    type Target = OwnedRow;
+
+    fn deref(&self) -> &Self::Target {
+        &self.row
+    }
+}
diff --git a/src/storage/src/write_batch.rs b/src/storage/src/write_batch.rs
index affbc5dbabf40..dc143f4a93855 100644
--- a/src/storage/src/write_batch.rs
+++ b/src/storage/src/write_batch.rs
@@ -22,7 +22,7 @@ use crate::hummock::HummockError;
 use crate::storage_value::StorageValue;
 use crate::store::{StateStoreWrite, WriteOptions};
 
-/// [`WriteBatch`] wraps a list of key-value pairs and an associated [`StateStore`].
+/// [`WriteBatch`] wraps a list of key-value pairs and an associated [`crate::StateStore`].
 pub struct WriteBatch<'a, S: StateStoreWrite> {
     store: &'a S,
 
diff --git a/src/stream/Cargo.toml b/src/stream/Cargo.toml
index e86b8913a6e7a..940b5191a4914 100644
--- a/src/stream/Cargo.toml
+++ b/src/stream/Cargo.toml
@@ -27,6 +27,7 @@ either = "1"
 enum-as-inner = "0.6"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
+governor = { version = "0.6", default-features = false, features = ["std", "dashmap", "jitter"] }
 hytra = "0.1.2"
 iter-chunks = "0.1"
 itertools = "0.11"
@@ -37,6 +38,7 @@ memcomparable = "0.2"
 multimap = "0.8"
 num-traits = "0.2"
 parking_lot = "0.12"
+parse-display = "0.8"
 pin-project = "1"
 prometheus = { version = "0.13", features = ["process"] }
 prost = "0.11"
diff --git a/src/stream/src/cache/mod.rs b/src/stream/src/cache/mod.rs
index 4bba736430e2b..ff3c214a1f5fd 100644
--- a/src/stream/src/cache/mod.rs
+++ b/src/stream/src/cache/mod.rs
@@ -26,7 +26,7 @@ use risingwave_common::util::iter_util::ZipEqFast;
 /// cache aware of the consistent hashing to avoid unnecessary eviction in the future.
 /// Check this [issue](https://github.com/risingwavelabs/risingwave/issues/5567).
 ///
-/// TODO: may encapsulate the logic into [`ExecutorCache`] when ready.
+/// TODO: may encapsulate the logic into `ExecutorCache` when ready.
 ///
 /// # Explanation
 /// We use a lazy manner to manipulate the cache. When scaling out, the partition of the existing
diff --git a/src/stream/src/common/builder.rs b/src/stream/src/common/builder.rs
index 0de5b64755315..0945066592d83 100644
--- a/src/stream/src/common/builder.rs
+++ b/src/stream/src/common/builder.rs
@@ -12,15 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::array::stream_record::Record;
 use risingwave_common::array::{ArrayBuilderImpl, Op, StreamChunk};
 use risingwave_common::row::Row;
-use risingwave_common::types::{DataType, Datum};
+use risingwave_common::types::{DataType, DatumRef};
 use risingwave_common::util::iter_util::ZipEqFast;
 
-type IndexMappings = Vec<(usize, usize)>;
-
-// FIXME(rc): I think this type is over-designed for executors with two side inputs.
-/// Build a array and it's corresponding operations.
+/// Build stream chunks with fixed chunk size from rows or records.
 pub struct StreamChunkBuilder {
     /// operations in the data chunk to build
     ops: Vec<Op>,
@@ -31,12 +29,6 @@ pub struct StreamChunkBuilder {
     /// Data types of columns
     data_types: Vec<DataType>,
 
-    /// The column index mapping from update side to output.
-    update_to_output: IndexMappings,
-
-    /// The column index mapping from matched side to output.
-    matched_to_output: IndexMappings,
-
     /// Maximum capacity of column builder
     capacity: usize,
 
@@ -57,56 +49,23 @@ impl Drop for StreamChunkBuilder {
 }
 
 impl StreamChunkBuilder {
-    pub fn new(
-        capacity: usize,
-        data_types: &[DataType],
-        update_to_output: IndexMappings,
-        matched_to_output: IndexMappings,
-    ) -> Self {
-        // Leave room for paired `UpdateDelete` and `UpdateInsert`. When there are `capacity - 1`
-        // ops in current builder and the last op is `UpdateDelete`, we delay the chunk generation
-        // until `UpdateInsert` comes. This means that the effective output message size will indeed
-        // be at most the original `capacity`
-        let reduced_capacity = capacity - 1;
-        assert!(reduced_capacity > 0);
-
-        let ops = Vec::with_capacity(capacity);
+    pub fn new(chunk_size: usize, data_types: Vec<DataType>) -> Self {
+        assert!(chunk_size > 0);
+
+        let ops = Vec::with_capacity(chunk_size);
         let column_builders = data_types
             .iter()
-            .map(|datatype| datatype.create_array_builder(capacity))
+            .map(|datatype| datatype.create_array_builder(chunk_size))
             .collect();
         Self {
             ops,
             column_builders,
-            data_types: data_types.to_owned(),
-            update_to_output,
-            matched_to_output,
-            capacity: reduced_capacity,
+            data_types,
+            capacity: chunk_size,
             size: 0,
         }
     }
 
-    /// Get the mapping from left/right input indices to the output indices.
-    pub fn get_i2o_mapping(
-        output_indices: impl Iterator<Item = usize>,
-        left_len: usize,
-        right_len: usize,
-    ) -> (IndexMappings, IndexMappings) {
-        let mut left_to_output = vec![];
-        let mut right_to_output = vec![];
-
-        for (output_idx, idx) in output_indices.enumerate() {
-            if idx < left_len {
-                left_to_output.push((idx, output_idx))
-            } else if idx >= left_len && idx < left_len + right_len {
-                right_to_output.push((idx - left_len, output_idx));
-            } else {
-                unreachable!("output_indices out of bound")
-            }
-        }
-        (left_to_output, right_to_output)
-    }
-
     /// Increase chunk size
     ///
     /// A [`StreamChunk`] will be returned when `size == capacity`
@@ -114,8 +73,10 @@ impl StreamChunkBuilder {
     fn inc_size(&mut self) -> Option<StreamChunk> {
         self.size += 1;
 
-        // Take a chunk when capacity is exceeded, but splitting `UpdateDelete` and `UpdateInsert`
-        // should be avoided
+        // Take a chunk when capacity is exceeded. Splitting `UpdateDelete` and `UpdateInsert`
+        // should be avoided, so when the last one is `UpdateDelete`, we delay the chunk until
+        // `UpdateInsert` comes. This means the output chunk size may exceed the given `chunk_size`,
+        // and theoretically at most `chunk_size + 1` if inputs are consistent.
         if self.size >= self.capacity && self.ops[self.ops.len() - 1] != Op::UpdateDelete {
             self.take()
         } else {
@@ -123,57 +84,39 @@ impl StreamChunkBuilder {
         }
     }
 
-    /// Append a row with coming update value and matched value
-    ///
-    /// A [`StreamChunk`] will be returned when `size == capacity`
-    #[must_use]
-    pub fn append_row(
+    /// Append an iterator of output index and datum to the builder, return a chunk if the builder
+    /// is full.
+    /// Note: the caller must ensure that each column occurs exactly once in `iter`.
+    fn append_iter<'a>(
         &mut self,
         op: Op,
-        row_update: impl Row,
-        row_matched: impl Row,
+        iter: impl IntoIterator<Item = (usize, DatumRef<'a>)>,
     ) -> Option<StreamChunk> {
         self.ops.push(op);
-        for &(update_idx, output_idx) in &self.update_to_output {
-            self.column_builders[output_idx].append(row_update.datum_at(update_idx));
+        for (i, datum) in iter {
+            self.column_builders[i].append(datum);
         }
-        for &(matched_idx, output_idx) in &self.matched_to_output {
-            self.column_builders[output_idx].append(row_matched.datum_at(matched_idx));
-        }
-
         self.inc_size()
     }
 
-    /// Append a row with coming update value and fill the other side with null.
-    ///
-    /// A [`StreamChunk`] will be returned when `size == capacity`
+    /// Append a row to the builder, return a chunk if the builder is full.
     #[must_use]
-    pub fn append_row_update(&mut self, op: Op, row_update: impl Row) -> Option<StreamChunk> {
-        self.ops.push(op);
-        for &(update_idx, output_idx) in &self.update_to_output {
-            self.column_builders[output_idx].append(row_update.datum_at(update_idx));
-        }
-        for &(_matched_idx, output_idx) in &self.matched_to_output {
-            self.column_builders[output_idx].append(Datum::None);
-        }
-
-        self.inc_size()
+    pub fn append_row(&mut self, op: Op, row: impl Row) -> Option<StreamChunk> {
+        self.append_iter(op, row.iter().enumerate())
     }
 
-    /// append a row with matched value and fill the coming side with null.
-    ///
-    /// A [`StreamChunk`] will be returned when `size == capacity`
+    /// Append a record to the builder, return a chunk if the builder is full.
     #[must_use]
-    pub fn append_row_matched(&mut self, op: Op, row_matched: impl Row) -> Option<StreamChunk> {
-        self.ops.push(op);
-        for &(_update_idx, output_idx) in &self.update_to_output {
-            self.column_builders[output_idx].append(Datum::None);
-        }
-        for &(matched_idx, output_idx) in &self.matched_to_output {
-            self.column_builders[output_idx].append(row_matched.datum_at(matched_idx));
+    pub fn append_record(&mut self, record: Record<impl Row>) -> Option<StreamChunk> {
+        match record {
+            Record::Insert { new_row } => self.append_row(Op::Insert, new_row),
+            Record::Delete { old_row } => self.append_row(Op::Delete, old_row),
+            Record::Update { old_row, new_row } => {
+                let none = self.append_row(Op::UpdateDelete, old_row);
+                debug_assert!(none.is_none());
+                self.append_row(Op::UpdateInsert, new_row)
+            }
         }
-
-        self.inc_size()
     }
 
     #[must_use]
@@ -194,9 +137,133 @@ impl StreamChunkBuilder {
             .collect::<Vec<_>>();
 
         Some(StreamChunk::new(
-            std::mem::take(&mut self.ops),
+            std::mem::replace(&mut self.ops, Vec::with_capacity(self.capacity)),
             new_columns,
             None,
         ))
     }
 }
+
+type IndexMappings = Vec<(usize, usize)>;
+
+/// Build stream chunks with fixed chunk size from joined two sides of rows.
+pub struct JoinStreamChunkBuilder {
+    builder: StreamChunkBuilder,
+
+    /// The column index mapping from update side to output.
+    update_to_output: IndexMappings,
+
+    /// The column index mapping from matched side to output.
+    matched_to_output: IndexMappings,
+}
+
+impl JoinStreamChunkBuilder {
+    pub fn new(
+        chunk_size: usize,
+        data_types: Vec<DataType>,
+        update_to_output: IndexMappings,
+        matched_to_output: IndexMappings,
+    ) -> Self {
+        Self {
+            builder: StreamChunkBuilder::new(chunk_size, data_types),
+            update_to_output,
+            matched_to_output,
+        }
+    }
+
+    /// Get the mappings from left/right input indices to the output indices. The mappings can be
+    /// used to create [`JoinStreamChunkBuilder`] later.
+    ///
+    /// Please note the semantics of `update` and `matched` when creating the builder: either left
+    /// or right side can be `update` side or `matched` side, the key is to call the corresponding
+    /// append method once you passed `left_to_output`/`right_to_output` to
+    /// `update_to_output`/`matched_to_output`.
+    pub fn get_i2o_mapping(
+        output_indices: &[usize],
+        left_len: usize,
+        right_len: usize,
+    ) -> (IndexMappings, IndexMappings) {
+        let mut left_to_output = vec![];
+        let mut right_to_output = vec![];
+
+        for (output_idx, &idx) in output_indices.iter().enumerate() {
+            if idx < left_len {
+                left_to_output.push((idx, output_idx))
+            } else if idx >= left_len && idx < left_len + right_len {
+                right_to_output.push((idx - left_len, output_idx));
+            } else {
+                unreachable!("output_indices out of bound")
+            }
+        }
+        (left_to_output, right_to_output)
+    }
+
+    /// Append a row with coming update value and matched value.
+    ///
+    /// A [`StreamChunk`] will be returned when `size == capacity`.
+    #[must_use]
+    pub fn append_row(
+        &mut self,
+        op: Op,
+        row_update: impl Row,
+        row_matched: impl Row,
+    ) -> Option<StreamChunk> {
+        self.builder.append_iter(
+            op,
+            self.update_to_output
+                .iter()
+                .map(|&(update_idx, output_idx)| (output_idx, row_update.datum_at(update_idx)))
+                .chain(
+                    self.matched_to_output
+                        .iter()
+                        .map(|&(matched_idx, output_idx)| {
+                            (output_idx, row_matched.datum_at(matched_idx))
+                        }),
+                ),
+        )
+    }
+
+    /// Append a row with coming update value and fill the other side with null.
+    ///
+    /// A [`StreamChunk`] will be returned when `size == capacity`.
+    #[must_use]
+    pub fn append_row_update(&mut self, op: Op, row_update: impl Row) -> Option<StreamChunk> {
+        self.builder.append_iter(
+            op,
+            self.update_to_output
+                .iter()
+                .map(|&(update_idx, output_idx)| (output_idx, row_update.datum_at(update_idx)))
+                .chain(
+                    self.matched_to_output
+                        .iter()
+                        .map(|&(_, output_idx)| (output_idx, DatumRef::None)),
+                ),
+        )
+    }
+
+    /// Append a row with matched value and fill the coming side with null.
+    ///
+    /// A [`StreamChunk`] will be returned when `size == capacity`.
+    #[must_use]
+    pub fn append_row_matched(&mut self, op: Op, row_matched: impl Row) -> Option<StreamChunk> {
+        self.builder.append_iter(
+            op,
+            self.update_to_output
+                .iter()
+                .map(|&(_, output_idx)| (output_idx, DatumRef::None))
+                .chain(
+                    self.matched_to_output
+                        .iter()
+                        .map(|&(matched_idx, output_idx)| {
+                            (output_idx, row_matched.datum_at(matched_idx))
+                        }),
+                ),
+        )
+    }
+
+    /// Take out the remaining rows as a chunk. Return `None` if the builder is empty.
+    #[must_use]
+    pub fn take(&mut self) -> Option<StreamChunk> {
+        self.builder.take()
+    }
+}
diff --git a/src/stream/src/common/log_store/in_mem.rs b/src/stream/src/common/log_store/in_mem.rs
index dec5eb2280f5b..f387d971e8ffe 100644
--- a/src/stream/src/common/log_store/in_mem.rs
+++ b/src/stream/src/common/log_store/in_mem.rs
@@ -12,13 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::future::Future;
 use std::sync::Arc;
 
 use anyhow::anyhow;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
-use risingwave_common::util::epoch::INVALID_EPOCH;
+use risingwave_common::util::epoch::{EpochPair, INVALID_EPOCH};
 use tokio::sync::mpsc::{
     channel, unbounded_channel, Receiver, Sender, UnboundedReceiver, UnboundedSender,
 };
@@ -35,6 +34,7 @@ enum InMemLogStoreItem {
         next_epoch: u64,
         is_checkpoint: bool,
     },
+    UpdateVnodeBitmap(Arc<Bitmap>),
 }
 
 /// An in-memory log store that can buffer a bounded amount of stream chunk in memory via bounded
@@ -95,174 +95,161 @@ impl LogStoreFactory for BoundedInMemLogStoreFactory {
     type Reader = BoundedInMemLogStoreReader;
     type Writer = BoundedInMemLogStoreWriter;
 
-    type BuildFuture = impl Future<Output = (Self::Reader, Self::Writer)>;
-
-    fn build(self) -> Self::BuildFuture {
-        async move {
-            let (init_epoch_tx, init_epoch_rx) = oneshot::channel();
-            let (item_tx, item_rx) = channel(self.bound);
-            let (truncated_epoch_tx, truncated_epoch_rx) = unbounded_channel();
-            let reader = BoundedInMemLogStoreReader {
-                epoch_progress: UNINITIALIZED,
-                init_epoch_rx: Some(init_epoch_rx),
-                item_rx,
-                truncated_epoch_tx,
-            };
-            let writer = BoundedInMemLogStoreWriter {
-                curr_epoch: None,
-                init_epoch_tx: Some(init_epoch_tx),
-                item_tx,
-                truncated_epoch_rx,
-            };
-            (reader, writer)
-        }
+    #[expect(clippy::unused_async)]
+    async fn build(self) -> (Self::Reader, Self::Writer) {
+        let (init_epoch_tx, init_epoch_rx) = oneshot::channel();
+        let (item_tx, item_rx) = channel(self.bound);
+        let (truncated_epoch_tx, truncated_epoch_rx) = unbounded_channel();
+        let reader = BoundedInMemLogStoreReader {
+            epoch_progress: UNINITIALIZED,
+            init_epoch_rx: Some(init_epoch_rx),
+            item_rx,
+            truncated_epoch_tx,
+        };
+        let writer = BoundedInMemLogStoreWriter {
+            curr_epoch: None,
+            init_epoch_tx: Some(init_epoch_tx),
+            item_tx,
+            truncated_epoch_rx,
+        };
+        (reader, writer)
     }
 }
 
 impl LogReader for BoundedInMemLogStoreReader {
-    type InitFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-    type NextItemFuture<'a> = impl Future<Output = LogStoreResult<(u64, LogStoreReadItem)>> + 'a;
-    type TruncateFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-
-    fn init(&mut self) -> Self::InitFuture<'_> {
-        async {
-            let init_epoch_rx = self
-                .init_epoch_rx
-                .take()
-                .expect("should not init for twice");
-            let epoch = init_epoch_rx
-                .await
-                .map_err(|e| anyhow!("unable to get init epoch: {:?}", e))?;
-            assert_eq!(self.epoch_progress, UNINITIALIZED);
-            self.epoch_progress = LogReaderEpochProgress::Consuming(epoch);
-            Ok(())
-        }
+    async fn init(&mut self) -> LogStoreResult<()> {
+        let init_epoch_rx = self
+            .init_epoch_rx
+            .take()
+            .expect("should not init for twice");
+        let epoch = init_epoch_rx
+            .await
+            .map_err(|e| anyhow!("unable to get init epoch: {:?}", e))?;
+        assert_eq!(self.epoch_progress, UNINITIALIZED);
+        self.epoch_progress = LogReaderEpochProgress::Consuming(epoch);
+        Ok(())
     }
 
-    fn next_item(&mut self) -> Self::NextItemFuture<'_> {
-        async {
-            match self.item_rx.recv().await {
-                Some(item) => match self.epoch_progress {
-                    Consuming(current_epoch) => match item {
-                        InMemLogStoreItem::StreamChunk(chunk) => {
-                            Ok((current_epoch, LogStoreReadItem::StreamChunk(chunk)))
-                        }
-                        InMemLogStoreItem::Barrier {
-                            is_checkpoint,
-                            next_epoch,
-                        } => {
-                            if is_checkpoint {
-                                self.epoch_progress = AwaitingTruncate {
-                                    next_epoch,
-                                    sealed_epoch: current_epoch,
-                                };
-                            } else {
-                                self.epoch_progress = Consuming(next_epoch);
-                            }
-                            Ok((current_epoch, LogStoreReadItem::Barrier { is_checkpoint }))
+    async fn next_item(&mut self) -> LogStoreResult<(u64, LogStoreReadItem)> {
+        match self.item_rx.recv().await {
+            Some(item) => match self.epoch_progress {
+                Consuming(current_epoch) => match item {
+                    InMemLogStoreItem::StreamChunk(chunk) => {
+                        Ok((current_epoch, LogStoreReadItem::StreamChunk(chunk)))
+                    }
+                    InMemLogStoreItem::Barrier {
+                        is_checkpoint,
+                        next_epoch,
+                    } => {
+                        if is_checkpoint {
+                            self.epoch_progress = AwaitingTruncate {
+                                next_epoch,
+                                sealed_epoch: current_epoch,
+                            };
+                        } else {
+                            self.epoch_progress = Consuming(next_epoch);
                         }
-                    },
-                    AwaitingTruncate { .. } => {
-                        unreachable!("should not be awaiting for when barrier comes")
+                        Ok((current_epoch, LogStoreReadItem::Barrier { is_checkpoint }))
                     }
+                    InMemLogStoreItem::UpdateVnodeBitmap(vnode_bitmap) => Ok((
+                        current_epoch,
+                        LogStoreReadItem::UpdateVnodeBitmap(vnode_bitmap),
+                    )),
                 },
-                None => Err(LogStoreError::EndOfLogStream),
-            }
+                AwaitingTruncate { .. } => {
+                    unreachable!("should not be awaiting for when barrier comes")
+                }
+            },
+            None => Err(LogStoreError::EndOfLogStream),
         }
     }
 
-    fn truncate(&mut self) -> Self::TruncateFuture<'_> {
-        async move {
-            let sealed_epoch = match self.epoch_progress {
-                Consuming(_) => unreachable!("should be awaiting truncate"),
-                AwaitingTruncate {
-                    sealed_epoch,
-                    next_epoch,
-                } => {
-                    self.epoch_progress = Consuming(next_epoch);
-                    sealed_epoch
-                }
-            };
-            self.truncated_epoch_tx
-                .send(sealed_epoch)
-                .map_err(|_| anyhow!("unable to send sealed epoch"))?;
-            Ok(())
-        }
+    #[expect(clippy::unused_async)]
+    async fn truncate(&mut self) -> LogStoreResult<()> {
+        let sealed_epoch = match self.epoch_progress {
+            Consuming(_) => unreachable!("should be awaiting truncate"),
+            AwaitingTruncate {
+                sealed_epoch,
+                next_epoch,
+            } => {
+                self.epoch_progress = Consuming(next_epoch);
+                sealed_epoch
+            }
+        };
+        self.truncated_epoch_tx
+            .send(sealed_epoch)
+            .map_err(|_| anyhow!("unable to send sealed epoch"))?;
+        Ok(())
     }
 }
 
 impl LogWriter for BoundedInMemLogStoreWriter {
-    type FlushCurrentEpoch<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-    type InitFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-    type WriteChunkFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-
-    fn init(&mut self, epoch: u64) -> Self::InitFuture<'_> {
-        async move {
-            let init_epoch_tx = self.init_epoch_tx.take().expect("cannot be init for twice");
-            init_epoch_tx
-                .send(epoch)
-                .map_err(|_| anyhow!("unable to send init epoch"))?;
-            self.curr_epoch = Some(epoch);
-            Ok(())
-        }
+    #[expect(clippy::unused_async)]
+    async fn init(&mut self, epoch: EpochPair) -> LogStoreResult<()> {
+        let init_epoch_tx = self.init_epoch_tx.take().expect("cannot be init for twice");
+        init_epoch_tx
+            .send(epoch.curr)
+            .map_err(|_| anyhow!("unable to send init epoch"))?;
+        self.curr_epoch = Some(epoch.curr);
+        Ok(())
     }
 
-    fn write_chunk(&mut self, chunk: StreamChunk) -> Self::WriteChunkFuture<'_> {
-        async {
-            self.item_tx
-                .send(InMemLogStoreItem::StreamChunk(chunk))
-                .await
-                .map_err(|_| anyhow!("unable to send stream chunk"))?;
-            Ok(())
-        }
+    async fn write_chunk(&mut self, chunk: StreamChunk) -> LogStoreResult<()> {
+        self.item_tx
+            .send(InMemLogStoreItem::StreamChunk(chunk))
+            .await
+            .map_err(|_| anyhow!("unable to send stream chunk"))?;
+        Ok(())
     }
 
-    fn flush_current_epoch(
+    async fn flush_current_epoch(
         &mut self,
         next_epoch: u64,
         is_checkpoint: bool,
-    ) -> Self::FlushCurrentEpoch<'_> {
-        async move {
-            self.item_tx
-                .send(InMemLogStoreItem::Barrier {
-                    next_epoch,
-                    is_checkpoint,
-                })
+    ) -> LogStoreResult<()> {
+        self.item_tx
+            .send(InMemLogStoreItem::Barrier {
+                next_epoch,
+                is_checkpoint,
+            })
+            .await
+            .map_err(|_| anyhow!("unable to send barrier"))?;
+
+        let prev_epoch = self
+            .curr_epoch
+            .replace(next_epoch)
+            .expect("should have epoch");
+
+        if is_checkpoint {
+            let truncated_epoch = self
+                .truncated_epoch_rx
+                .recv()
                 .await
-                .map_err(|_| anyhow!("unable to send barrier"))?;
-
-            let prev_epoch = self
-                .curr_epoch
-                .replace(next_epoch)
-                .expect("should have epoch");
-
-            if is_checkpoint {
-                let truncated_epoch = self
-                    .truncated_epoch_rx
-                    .recv()
-                    .await
-                    .ok_or_else(|| anyhow!("cannot get truncated epoch"))?;
-                assert_eq!(truncated_epoch, prev_epoch);
-            }
-
-            Ok(())
+                .ok_or_else(|| anyhow!("cannot get truncated epoch"))?;
+            assert_eq!(truncated_epoch, prev_epoch);
         }
+
+        Ok(())
     }
 
-    fn update_vnode_bitmap(&mut self, _new_vnodes: Arc<Bitmap>) {
-        // Since this is in memory, we don't need to handle the vnode bitmap
+    async fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) -> LogStoreResult<()> {
+        Ok(self
+            .item_tx
+            .send(InMemLogStoreItem::UpdateVnodeBitmap(new_vnodes))
+            .await
+            .map_err(|_| anyhow!("unable to send vnode bitmap"))?)
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::{Op, StreamChunk};
-    use risingwave_common::row::OwnedRow;
+    use risingwave_common::array::Op;
     use risingwave_common::types::{DataType, ScalarImpl};
-    use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+    use risingwave_common::util::epoch::EpochPair;
 
     use crate::common::log_store::in_mem::BoundedInMemLogStoreFactory;
     use crate::common::log_store::{LogReader, LogStoreFactory, LogStoreReadItem, LogWriter};
+    use crate::common::StreamChunkBuilder;
 
     #[tokio::test]
     async fn test_in_memory_log_store() {
@@ -274,21 +261,26 @@ mod tests {
         let epoch2 = init_epoch + 2;
 
         let ops = vec![Op::Insert, Op::Delete, Op::UpdateInsert, Op::UpdateDelete];
-        let mut builder = DataChunkBuilder::new(vec![DataType::Int64, DataType::Varchar], 10000);
-        for i in 0..ops.len() {
+        let mut builder = StreamChunkBuilder::new(10000, vec![DataType::Int64, DataType::Varchar]);
+        for (i, op) in ops.into_iter().enumerate() {
             assert!(builder
-                .append_one_row(OwnedRow::new(vec![
-                    Some(ScalarImpl::Int64(i as i64)),
-                    Some(ScalarImpl::Utf8(format!("name_{}", i).into_boxed_str()))
-                ]))
+                .append_row(
+                    op,
+                    [
+                        Some(ScalarImpl::Int64(i as i64)),
+                        Some(ScalarImpl::Utf8(format!("name_{}", i).into_boxed_str()))
+                    ]
+                )
                 .is_none());
         }
-        let data_chunk = builder.consume_all().unwrap();
-        let stream_chunk = StreamChunk::from_parts(ops, data_chunk);
+        let stream_chunk = builder.take().unwrap();
         let stream_chunk_clone = stream_chunk.clone();
 
         let join_handle = tokio::spawn(async move {
-            writer.init(init_epoch).await.unwrap();
+            writer
+                .init(EpochPair::new_test_epoch(init_epoch))
+                .await
+                .unwrap();
             writer
                 .write_chunk(stream_chunk_clone.clone())
                 .await
diff --git a/src/stream/src/common/log_store/kv_log_store/mod.rs b/src/stream/src/common/log_store/kv_log_store/mod.rs
index 3e5b87aaff917..f54ef44c49141 100644
--- a/src/stream/src/common/log_store/kv_log_store/mod.rs
+++ b/src/stream/src/common/log_store/kv_log_store/mod.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::future::Future;
 use std::sync::Arc;
 
 use risingwave_common::buffer::Bitmap;
@@ -72,39 +71,36 @@ impl<S: StateStore> LogStoreFactory for KvLogStoreFactory<S> {
     type Reader = KvLogStoreReader<S>;
     type Writer = KvLogStoreWriter<S::Local>;
 
-    type BuildFuture = impl Future<Output = (Self::Reader, Self::Writer)>;
-
-    fn build(self) -> Self::BuildFuture {
-        async move {
-            let table_id = TableId::new(self.table_catalog.id);
-            let serde = LogStoreRowSerde::new(&self.table_catalog, self.vnodes);
-            let local_state_store = self
-                .state_store
-                .new_local(NewLocalOptions {
-                    table_id: TableId {
-                        table_id: self.table_catalog.id,
-                    },
-                    is_consistent_op: false,
-                    table_option: TableOption {
-                        retention_seconds: None,
-                    },
-                    is_replicated: false,
-                })
-                .await;
-
-            let (tx, rx) = new_log_store_buffer(self.max_stream_chunk_count);
-
-            let reader = KvLogStoreReader::new(table_id, self.state_store, serde.clone(), rx);
-
-            let writer = KvLogStoreWriter::new(table_id, local_state_store, serde, tx);
-
-            (reader, writer)
-        }
+    async fn build(self) -> (Self::Reader, Self::Writer) {
+        let table_id = TableId::new(self.table_catalog.id);
+        let serde = LogStoreRowSerde::new(&self.table_catalog, self.vnodes);
+        let local_state_store = self
+            .state_store
+            .new_local(NewLocalOptions {
+                table_id: TableId {
+                    table_id: self.table_catalog.id,
+                },
+                is_consistent_op: false,
+                table_option: TableOption {
+                    retention_seconds: None,
+                },
+                is_replicated: false,
+            })
+            .await;
+
+        let (tx, rx) = new_log_store_buffer(self.max_stream_chunk_count);
+
+        let reader = KvLogStoreReader::new(table_id, self.state_store, serde.clone(), rx);
+
+        let writer = KvLogStoreWriter::new(table_id, local_state_store, serde, tx);
+
+        (reader, writer)
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use risingwave_common::util::epoch::EpochPair;
     use risingwave_hummock_sdk::HummockReadEpoch;
     use risingwave_hummock_test::test_utils::prepare_hummock_test_env;
     use risingwave_storage::store::SyncResult;
@@ -147,7 +143,10 @@ mod tests {
             .version()
             .max_committed_epoch
             + 1;
-        writer.init(epoch1).await.unwrap();
+        writer
+            .init(EpochPair::new_test_epoch(epoch1))
+            .await
+            .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1 + 1;
         writer.flush_current_epoch(epoch2, false).await.unwrap();
@@ -222,7 +221,10 @@ mod tests {
             .version()
             .max_committed_epoch
             + 1;
-        writer.init(epoch1).await.unwrap();
+        writer
+            .init(EpochPair::new_test_epoch(epoch1))
+            .await
+            .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1 + 1;
         writer.flush_current_epoch(epoch2, false).await.unwrap();
@@ -282,7 +284,10 @@ mod tests {
             max_stream_chunk_count,
         );
         let (mut reader, mut writer) = factory.build().await;
-        writer.init(epoch3).await.unwrap();
+        writer
+            .init(EpochPair::new_test_epoch(epoch3))
+            .await
+            .unwrap();
         reader.init().await.unwrap();
         match reader.next_item().await.unwrap() {
             (epoch, LogStoreReadItem::StreamChunk(read_stream_chunk)) => {
@@ -345,7 +350,10 @@ mod tests {
             .version()
             .max_committed_epoch
             + 1;
-        writer.init(epoch1).await.unwrap();
+        writer
+            .init(EpochPair::new_test_epoch(epoch1))
+            .await
+            .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1 + 1;
         writer.flush_current_epoch(epoch2, true).await.unwrap();
@@ -409,7 +417,10 @@ mod tests {
         );
         let (mut reader, mut writer) = factory.build().await;
 
-        writer.init(epoch3).await.unwrap();
+        writer
+            .init(EpochPair::new_test_epoch(epoch3))
+            .await
+            .unwrap();
         let stream_chunk3 = gen_stream_chunk(20);
         writer.write_chunk(stream_chunk3.clone()).await.unwrap();
 
diff --git a/src/stream/src/common/log_store/kv_log_store/reader.rs b/src/stream/src/common/log_store/kv_log_store/reader.rs
index 4ab1f174b0665..ffed40d9ce47b 100644
--- a/src/stream/src/common/log_store/kv_log_store/reader.rs
+++ b/src/stream/src/common/log_store/kv_log_store/reader.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::future::Future;
 use std::ops::Bound::{Excluded, Included};
 use std::pin::Pin;
 
@@ -75,154 +74,130 @@ impl<S: StateStore> KvLogStoreReader<S> {
 }
 
 impl<S: StateStore> LogReader for KvLogStoreReader<S> {
-    type InitFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-    type NextItemFuture<'a> = impl Future<Output = LogStoreResult<(u64, LogStoreReadItem)>> + 'a;
-    type TruncateFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
+    async fn init(&mut self) -> LogStoreResult<()> {
+        let first_write_epoch = self.rx.init().await;
+        let streams = try_join_all(self.serde.vnodes().iter_vnodes().map(|vnode| {
+            let range_start = Bytes::from(Vec::from(vnode.to_be_bytes()));
+            let range_end = self.serde.serialize_epoch(vnode, first_write_epoch);
+            let table_id = self.table_id;
+            let state_store = self.state_store.clone();
+            async move {
+                state_store
+                    .iter(
+                        (Included(range_start), Excluded(range_end)),
+                        u64::MAX,
+                        ReadOptions {
+                            prefetch_options: PrefetchOptions::new_for_exhaust_iter(),
+                            cache_policy: CachePolicy::Fill(CachePriority::Low),
+                            table_id,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+            }
+        }))
+        .await?;
+        // TODO: set chunk size by config
+        let state_store_stream =
+            Box::pin(new_log_store_item_stream(streams, self.serde.clone(), 1024));
+        self.reader_state = ReaderState::ConsumingStateStore {
+            first_write_epoch,
+            state_store_stream,
+        };
+        Ok(())
+    }
 
-    fn init(&mut self) -> Self::InitFuture<'_> {
-        async move {
-            let first_write_epoch = self.rx.init().await;
-            let streams = try_join_all(self.serde.vnodes().iter_vnodes().map(|vnode| {
-                let range_start = Bytes::from(Vec::from(vnode.to_be_bytes()));
-                let range_end = self.serde.serialize_epoch(vnode, first_write_epoch);
-                let table_id = self.table_id;
-                let state_store = self.state_store.clone();
-                async move {
-                    state_store
-                        .iter(
-                            (Included(range_start), Excluded(range_end)),
-                            u64::MAX,
-                            ReadOptions {
-                                prefix_hint: None,
-                                ignore_range_tombstone: false,
-                                prefetch_options: PrefetchOptions::new_for_exhaust_iter(),
-                                cache_policy: CachePolicy::Fill(CachePriority::Low),
-                                retention_seconds: None,
-                                table_id,
-                                read_version_from_backup: false,
-                            },
-                        )
-                        .await
-                }
-            }))
-            .await?;
-            // TODO: set chunk size by config
-            let state_store_stream =
-                Box::pin(new_log_store_item_stream(streams, self.serde.clone(), 1024));
-            self.reader_state = ReaderState::ConsumingStateStore {
+    async fn next_item(&mut self) -> LogStoreResult<(u64, LogStoreReadItem)> {
+        let epoch = match &mut self.reader_state {
+            ReaderState::Uninitialized => unreachable!("should be initialized"),
+            ReaderState::ConsumingStateStore {
                 first_write_epoch,
                 state_store_stream,
-            };
-            Ok(())
-        }
-    }
-
-    fn next_item(&mut self) -> Self::NextItemFuture<'_> {
-        async move {
-            let epoch = match &mut self.reader_state {
-                ReaderState::Uninitialized => unreachable!("should be initialized"),
-                ReaderState::ConsumingStateStore {
-                    first_write_epoch,
-                    state_store_stream,
-                } => {
-                    match state_store_stream.try_next().await? {
-                        Some((epoch, item)) => {
-                            return Ok((epoch, item));
-                        }
-                        None => {
-                            let first_write_epoch = *first_write_epoch;
-                            // all consumed
-                            self.reader_state = ReaderState::ConsumingStream {
-                                epoch: first_write_epoch,
-                            };
-                            first_write_epoch
-                        }
-                    }
-                }
-                ReaderState::ConsumingStream { epoch } => *epoch,
-            };
-            loop {
-                let (item_epoch, item) = self.rx.next_item().await;
-                assert_eq!(epoch, item_epoch);
-                match item {
-                    LogStoreBufferItem::StreamChunk { chunk, .. } => {
-                        return Ok((epoch, LogStoreReadItem::StreamChunk(chunk)));
-                    }
-                    LogStoreBufferItem::Flushed {
-                        vnode_bitmap,
-                        start_seq_id,
-                        end_seq_id,
-                    } => {
-                        let streams = try_join_all(vnode_bitmap.iter_vnodes().map(|vnode| {
-                            let range_start =
-                                self.serde
-                                    .serialize_log_store_pk(vnode, epoch, start_seq_id);
-                            let range_end =
-                                self.serde.serialize_log_store_pk(vnode, epoch, end_seq_id);
-                            let state_store = self.state_store.clone();
-                            let table_id = self.table_id;
-                            // Use u64::MAX here because the epoch to consume may be below the safe
-                            // epoch
-                            async move {
-                                Ok::<_, LogStoreError>(Box::pin(
-                                    state_store
-                                        .iter(
-                                            (Included(range_start), Included(range_end)),
-                                            u64::MAX,
-                                            ReadOptions {
-                                                prefix_hint: None,
-                                                ignore_range_tombstone: false,
-                                                prefetch_options:
-                                                    PrefetchOptions::new_for_exhaust_iter(),
-                                                cache_policy: CachePolicy::Fill(CachePriority::Low),
-                                                retention_seconds: None,
-                                                table_id,
-                                                read_version_from_backup: false,
-                                            },
-                                        )
-                                        .await?,
-                                ))
-                            }
-                        }))
-                        .await?;
-                        let combined_stream = select_all(streams);
-                        let stream_chunk = self
-                            .serde
-                            .deserialize_stream_chunk(
-                                combined_stream,
-                                start_seq_id,
-                                end_seq_id,
-                                epoch,
-                            )
-                            .await?;
-                        return Ok((epoch, LogStoreReadItem::StreamChunk(stream_chunk)));
+            } => {
+                match state_store_stream.try_next().await? {
+                    Some((epoch, item)) => {
+                        return Ok((epoch, item));
                     }
-                    LogStoreBufferItem::Barrier {
-                        is_checkpoint,
-                        next_epoch,
-                    } => {
-                        assert!(
-                            epoch < next_epoch,
-                            "next epoch {} should be greater than current epoch {}",
-                            next_epoch,
-                            epoch
-                        );
-                        self.reader_state = ReaderState::ConsumingStream { epoch: next_epoch };
-                        return Ok((epoch, LogStoreReadItem::Barrier { is_checkpoint }));
-                    }
-                    LogStoreBufferItem::UpdateVnodes(bitmap) => {
-                        self.serde.update_vnode_bitmap(bitmap);
-                        continue;
+                    None => {
+                        let first_write_epoch = *first_write_epoch;
+                        // all consumed
+                        self.reader_state = ReaderState::ConsumingStream {
+                            epoch: first_write_epoch,
+                        };
+                        first_write_epoch
                     }
                 }
             }
-        }
+            ReaderState::ConsumingStream { epoch } => *epoch,
+        };
+        let (item_epoch, item) = self.rx.next_item().await;
+        assert_eq!(epoch, item_epoch);
+        Ok(match item {
+            LogStoreBufferItem::StreamChunk { chunk, .. } => {
+                (epoch, LogStoreReadItem::StreamChunk(chunk))
+            }
+            LogStoreBufferItem::Flushed {
+                vnode_bitmap,
+                start_seq_id,
+                end_seq_id,
+            } => {
+                let streams = try_join_all(vnode_bitmap.iter_vnodes().map(|vnode| {
+                    let range_start = self
+                        .serde
+                        .serialize_log_store_pk(vnode, epoch, start_seq_id);
+                    let range_end = self.serde.serialize_log_store_pk(vnode, epoch, end_seq_id);
+                    let state_store = self.state_store.clone();
+                    let table_id = self.table_id;
+                    // Use u64::MAX here because the epoch to consume may be below the safe
+                    // epoch
+                    async move {
+                        Ok::<_, LogStoreError>(Box::pin(
+                            state_store
+                                .iter(
+                                    (Included(range_start), Included(range_end)),
+                                    u64::MAX,
+                                    ReadOptions {
+                                        prefetch_options: PrefetchOptions::new_for_exhaust_iter(),
+                                        cache_policy: CachePolicy::Fill(CachePriority::Low),
+                                        table_id,
+                                        ..Default::default()
+                                    },
+                                )
+                                .await?,
+                        ))
+                    }
+                }))
+                .await?;
+                let combined_stream = select_all(streams);
+                let stream_chunk = self
+                    .serde
+                    .deserialize_stream_chunk(combined_stream, start_seq_id, end_seq_id, epoch)
+                    .await?;
+                (epoch, LogStoreReadItem::StreamChunk(stream_chunk))
+            }
+            LogStoreBufferItem::Barrier {
+                is_checkpoint,
+                next_epoch,
+            } => {
+                assert!(
+                    epoch < next_epoch,
+                    "next epoch {} should be greater than current epoch {}",
+                    next_epoch,
+                    epoch
+                );
+                self.reader_state = ReaderState::ConsumingStream { epoch: next_epoch };
+                (epoch, LogStoreReadItem::Barrier { is_checkpoint })
+            }
+            LogStoreBufferItem::UpdateVnodes(bitmap) => {
+                self.serde.update_vnode_bitmap(bitmap.clone());
+                (epoch, LogStoreReadItem::UpdateVnodeBitmap(bitmap))
+            }
+        })
     }
 
-    fn truncate(&mut self) -> Self::TruncateFuture<'_> {
-        async move {
-            self.rx.truncate();
-            Ok(())
-        }
+    #[expect(clippy::unused_async)]
+    async fn truncate(&mut self) -> LogStoreResult<()> {
+        self.rx.truncate();
+        Ok(())
     }
 }
diff --git a/src/stream/src/common/log_store/kv_log_store/serde.rs b/src/stream/src/common/log_store/kv_log_store/serde.rs
index fd26bb2484c63..15825e9f275e6 100644
--- a/src/stream/src/common/log_store/kv_log_store/serde.rs
+++ b/src/stream/src/common/log_store/kv_log_store/serde.rs
@@ -944,7 +944,7 @@ mod tests {
                     assert_eq!(row.to_owned_row(), rows[i]);
                 }
             }
-            LogStoreReadItem::Barrier { .. } => unreachable!(),
+            _ => unreachable!(),
         }
 
         let (epoch, item): (_, LogStoreReadItem) = stream.try_next().await.unwrap().unwrap();
@@ -957,7 +957,7 @@ mod tests {
                     assert_eq!(row.to_owned_row(), rows[i + CHUNK_SIZE]);
                 }
             }
-            LogStoreReadItem::Barrier { .. } => unreachable!(),
+            _ => unreachable!(),
         }
 
         let (epoch, item): (_, LogStoreReadItem) = stream.try_next().await.unwrap().unwrap();
@@ -967,6 +967,7 @@ mod tests {
             LogStoreReadItem::Barrier { is_checkpoint } => {
                 assert!(!is_checkpoint);
             }
+            _ => unreachable!(),
         }
 
         assert!(poll_fn(|cx| Poll::Ready(stream.poll_next_unpin(cx)))
@@ -985,7 +986,7 @@ mod tests {
                     assert_eq!(row.to_owned_row(), rows[i]);
                 }
             }
-            LogStoreReadItem::Barrier { .. } => unreachable!(),
+            _ => unreachable!(),
         }
 
         let (epoch, item): (_, LogStoreReadItem) = stream.try_next().await.unwrap().unwrap();
@@ -998,7 +999,7 @@ mod tests {
                     assert_eq!(row.to_owned_row(), rows[i + CHUNK_SIZE]);
                 }
             }
-            LogStoreReadItem::Barrier { .. } => unreachable!(),
+            _ => unreachable!(),
         }
 
         let (epoch, item): (_, LogStoreReadItem) = stream.try_next().await.unwrap().unwrap();
@@ -1008,6 +1009,7 @@ mod tests {
             LogStoreReadItem::Barrier { is_checkpoint } => {
                 assert!(is_checkpoint);
             }
+            _ => unreachable!(),
         }
 
         assert!(stream.next().await.is_none());
diff --git a/src/stream/src/common/log_store/kv_log_store/writer.rs b/src/stream/src/common/log_store/kv_log_store/writer.rs
index acfeb6d6c80d2..2b46177be3985 100644
--- a/src/stream/src/common/log_store/kv_log_store/writer.rs
+++ b/src/stream/src/common/log_store/kv_log_store/writer.rs
@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::future::Future;
-use std::ops::Bound::Included;
+use std::ops::Bound::{Excluded, Included};
 use std::sync::Arc;
 
 use bytes::Bytes;
@@ -21,7 +20,8 @@ use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
-use risingwave_storage::store::LocalStateStore;
+use risingwave_common::util::epoch::EpochPair;
+use risingwave_storage::store::{InitOptions, LocalStateStore};
 
 use crate::common::log_store::kv_log_store::buffer::LogStoreBufferSender;
 use crate::common::log_store::kv_log_store::serde::LogStoreRowSerde;
@@ -58,91 +58,85 @@ impl<LS: LocalStateStore> KvLogStoreWriter<LS> {
 }
 
 impl<LS: LocalStateStore> LogWriter for KvLogStoreWriter<LS> {
-    type FlushCurrentEpoch<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-    type InitFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-    type WriteChunkFuture<'a> = impl Future<Output = LogStoreResult<()>> + 'a;
-
-    fn init(&mut self, epoch: u64) -> Self::InitFuture<'_> {
-        async move {
-            self.state_store.init(epoch);
-            self.seq_id = FIRST_SEQ_ID;
-            self.tx.init(epoch);
-            Ok(())
-        }
+    async fn init(&mut self, epoch: EpochPair) -> LogStoreResult<()> {
+        self.state_store
+            .init(InitOptions::new_with_epoch(epoch))
+            .await?;
+        self.seq_id = FIRST_SEQ_ID;
+        self.tx.init(epoch.curr);
+        Ok(())
     }
 
-    fn write_chunk(&mut self, chunk: StreamChunk) -> Self::WriteChunkFuture<'_> {
-        async move {
-            assert!(chunk.cardinality() > 0);
-            let epoch = self.state_store.epoch();
-            let start_seq_id = self.seq_id;
-            self.seq_id += chunk.cardinality() as SeqIdType;
-            let end_seq_id = self.seq_id - 1;
-            if let Some(chunk) =
-                self.tx
-                    .try_add_stream_chunk(epoch, chunk, start_seq_id, end_seq_id)
-            {
-                // When enter this branch, the chunk cannot be added directly, and should be add to
-                // state store and flush
-                let mut vnode_bitmap_builder = BitmapBuilder::zeroed(VirtualNode::COUNT);
-                for (i, (op, row)) in chunk.rows().enumerate() {
-                    let seq_id = start_seq_id + (i as SeqIdType);
-                    assert!(seq_id <= end_seq_id);
-                    let (vnode, key, value) = self.serde.serialize_data_row(epoch, seq_id, op, row);
-                    vnode_bitmap_builder.set(vnode.to_index(), true);
-                    self.state_store.insert(key, value, None)?;
-                }
-                self.state_store.flush(Vec::new()).await?;
-
-                let vnode_bitmap = vnode_bitmap_builder.finish();
-                self.tx
-                    .add_flushed(epoch, start_seq_id, end_seq_id, vnode_bitmap);
+    async fn write_chunk(&mut self, chunk: StreamChunk) -> LogStoreResult<()> {
+        assert!(chunk.cardinality() > 0);
+        let epoch = self.state_store.epoch();
+        let start_seq_id = self.seq_id;
+        self.seq_id += chunk.cardinality() as SeqIdType;
+        let end_seq_id = self.seq_id - 1;
+        if let Some(chunk) = self
+            .tx
+            .try_add_stream_chunk(epoch, chunk, start_seq_id, end_seq_id)
+        {
+            // When enter this branch, the chunk cannot be added directly, and should be add to
+            // state store and flush
+            let mut vnode_bitmap_builder = BitmapBuilder::zeroed(VirtualNode::COUNT);
+            for (i, (op, row)) in chunk.rows().enumerate() {
+                let seq_id = start_seq_id + (i as SeqIdType);
+                assert!(seq_id <= end_seq_id);
+                let (vnode, key, value) = self.serde.serialize_data_row(epoch, seq_id, op, row);
+                vnode_bitmap_builder.set(vnode.to_index(), true);
+                self.state_store.insert(key, value, None)?;
             }
-            Ok(())
+            self.state_store.flush(Vec::new()).await?;
+
+            let vnode_bitmap = vnode_bitmap_builder.finish();
+            self.tx
+                .add_flushed(epoch, start_seq_id, end_seq_id, vnode_bitmap);
         }
+        Ok(())
     }
 
-    fn flush_current_epoch(
+    async fn flush_current_epoch(
         &mut self,
         next_epoch: u64,
         is_checkpoint: bool,
-    ) -> Self::FlushCurrentEpoch<'_> {
-        async move {
-            let epoch = self.state_store.epoch();
-            for vnode in self.serde.vnodes().iter_vnodes() {
-                let (key, value) = self.serde.serialize_barrier(epoch, vnode, is_checkpoint);
-                self.state_store.insert(key, value, None)?;
-            }
-            self.tx
-                .flush_all_unflushed(|chunk, epoch, start_seq_id, end_seq_id| {
-                    for (i, (op, row)) in chunk.rows().enumerate() {
-                        let seq_id = start_seq_id + (i as SeqIdType);
-                        assert!(seq_id <= end_seq_id);
-                        let (_, key, value) = self.serde.serialize_data_row(epoch, seq_id, op, row);
-                        self.state_store.insert(key, value, None)?;
-                    }
-                    Ok(())
-                })?;
-            let mut delete_range = Vec::with_capacity(self.serde.vnodes().count_ones());
-            if let Some(truncation_offset) = self.tx.pop_truncation() {
-                for vnode in self.serde.vnodes().iter_vnodes() {
-                    let range_begin = Bytes::from(vnode.to_be_bytes().to_vec());
-                    let range_end = self
-                        .serde
-                        .serialize_truncation_offset_watermark(vnode, truncation_offset);
-                    delete_range.push((Included(range_begin), Included(range_end)));
+    ) -> LogStoreResult<()> {
+        let epoch = self.state_store.epoch();
+        for vnode in self.serde.vnodes().iter_vnodes() {
+            let (key, value) = self.serde.serialize_barrier(epoch, vnode, is_checkpoint);
+            self.state_store.insert(key, value, None)?;
+        }
+        self.tx
+            .flush_all_unflushed(|chunk, epoch, start_seq_id, end_seq_id| {
+                for (i, (op, row)) in chunk.rows().enumerate() {
+                    let seq_id = start_seq_id + (i as SeqIdType);
+                    assert!(seq_id <= end_seq_id);
+                    let (_, key, value) = self.serde.serialize_data_row(epoch, seq_id, op, row);
+                    self.state_store.insert(key, value, None)?;
                 }
+                Ok(())
+            })?;
+        let mut delete_range = Vec::with_capacity(self.serde.vnodes().count_ones());
+        if let Some(truncation_offset) = self.tx.pop_truncation() {
+            for vnode in self.serde.vnodes().iter_vnodes() {
+                let range_begin = Bytes::from(vnode.to_be_bytes().to_vec());
+                let range_end = self
+                    .serde
+                    .serialize_truncation_offset_watermark(vnode, truncation_offset);
+                delete_range.push((Included(range_begin), Excluded(range_end)));
             }
-            self.state_store.flush(delete_range).await?;
-            self.state_store.seal_current_epoch(next_epoch);
-            self.tx.barrier(epoch, is_checkpoint, next_epoch);
-            self.seq_id = FIRST_SEQ_ID;
-            Ok(())
         }
+        self.state_store.flush(delete_range).await?;
+        self.state_store.seal_current_epoch(next_epoch);
+        self.tx.barrier(epoch, is_checkpoint, next_epoch);
+        self.seq_id = FIRST_SEQ_ID;
+        Ok(())
     }
 
-    fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) {
+    #[expect(clippy::unused_async)]
+    async fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) -> LogStoreResult<()> {
         self.serde.update_vnode_bitmap(new_vnodes.clone());
         self.tx.update_vnode(self.state_store.epoch(), new_vnodes);
+        Ok(())
     }
 }
diff --git a/src/stream/src/common/log_store/mod.rs b/src/stream/src/common/log_store/mod.rs
index 058f462183a1f..f343cfdfc8f03 100644
--- a/src/stream/src/common/log_store/mod.rs
+++ b/src/stream/src/common/log_store/mod.rs
@@ -21,6 +21,7 @@ use std::sync::Arc;
 
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
+use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::value_encoding::error::ValueEncodingError;
 use risingwave_storage::error::StorageError;
 
@@ -45,63 +46,50 @@ pub type LogStoreResult<T> = Result<T, LogStoreError>;
 pub enum LogStoreReadItem {
     StreamChunk(StreamChunk),
     Barrier { is_checkpoint: bool },
+    UpdateVnodeBitmap(Arc<Bitmap>),
 }
 
 pub trait LogWriter {
-    type InitFuture<'a>: Future<Output = LogStoreResult<()>> + Send + 'a
-    where
-        Self: 'a;
-    type WriteChunkFuture<'a>: Future<Output = LogStoreResult<()>> + Send + 'a
-    where
-        Self: 'a;
-    type FlushCurrentEpoch<'a>: Future<Output = LogStoreResult<()>> + Send + 'a
-    where
-        Self: 'a;
-
     /// Initialize the log writer with an epoch
-    fn init(&mut self, epoch: u64) -> Self::InitFuture<'_>;
+    fn init(&mut self, epoch: EpochPair) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
 
     /// Write a stream chunk to the log writer
-    fn write_chunk(&mut self, chunk: StreamChunk) -> Self::WriteChunkFuture<'_>;
+    fn write_chunk(
+        &mut self,
+        chunk: StreamChunk,
+    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
 
     /// Mark current epoch as finished and sealed, and flush the unconsumed log data.
     fn flush_current_epoch(
         &mut self,
         next_epoch: u64,
         is_checkpoint: bool,
-    ) -> Self::FlushCurrentEpoch<'_>;
+    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
 
     /// Update the vnode bitmap of the log writer
-    fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>);
+    fn update_vnode_bitmap(
+        &mut self,
+        new_vnodes: Arc<Bitmap>,
+    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
 }
 
 pub trait LogReader {
-    type InitFuture<'a>: Future<Output = LogStoreResult<()>> + Send + 'a
-    where
-        Self: 'a;
-    type NextItemFuture<'a>: Future<Output = LogStoreResult<(u64, LogStoreReadItem)>> + Send + 'a
-    where
-        Self: 'a;
-    type TruncateFuture<'a>: Future<Output = LogStoreResult<()>> + Send + 'a
-    where
-        Self: 'a;
-
     /// Initialize the log reader. Usually function as waiting for log writer to be initialized.
-    fn init(&mut self) -> Self::InitFuture<'_>;
+    fn init(&mut self) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
 
     /// Emit the next item.
-    fn next_item(&mut self) -> Self::NextItemFuture<'_>;
+    fn next_item(
+        &mut self,
+    ) -> impl Future<Output = LogStoreResult<(u64, LogStoreReadItem)>> + Send + '_;
 
     /// Mark that all items emitted so far have been consumed and it is safe to truncate the log
     /// from the current offset.
-    fn truncate(&mut self) -> Self::TruncateFuture<'_>;
+    fn truncate(&mut self) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
 }
 
 pub trait LogStoreFactory: 'static {
     type Reader: LogReader + Send + 'static;
     type Writer: LogWriter + Send + 'static;
 
-    type BuildFuture: Future<Output = (Self::Reader, Self::Writer)> + Send;
-
-    fn build(self) -> Self::BuildFuture;
+    fn build(self) -> impl Future<Output = (Self::Reader, Self::Writer)> + Send;
 }
diff --git a/src/stream/src/common/table/state_table.rs b/src/stream/src/common/table/state_table.rs
index 28f36e8dc908a..dfdd11732c942 100644
--- a/src/stream/src/common/table/state_table.rs
+++ b/src/stream/src/common/table/state_table.rs
@@ -17,7 +17,7 @@ use std::ops::Bound::*;
 use std::sync::Arc;
 
 use bytes::{BufMut, Bytes, BytesMut};
-use futures::{pin_mut, Stream, StreamExt};
+use futures::{pin_mut, FutureExt, Stream, StreamExt};
 use futures_async_stream::for_await;
 use itertools::{izip, Itertools};
 use risingwave_common::array::stream_record::Record;
@@ -37,7 +37,7 @@ use risingwave_hummock_sdk::key::{
     end_bound_of_prefix, next_key, prefixed_range, range_of_prefix, start_bound_of_excluded_prefix,
 };
 use risingwave_pb::catalog::Table;
-use risingwave_storage::error::StorageError;
+use risingwave_storage::error::{StorageError, StorageResult};
 use risingwave_storage::hummock::CachePolicy;
 use risingwave_storage::mem_table::MemTableError;
 use risingwave_storage::row_serde::row_serde_util::{
@@ -45,10 +45,11 @@ use risingwave_storage::row_serde::row_serde_util::{
 };
 use risingwave_storage::row_serde::value_serde::ValueRowSerde;
 use risingwave_storage::store::{
-    LocalStateStore, NewLocalOptions, PrefetchOptions, ReadOptions, StateStoreIterItemStream,
+    InitOptions, LocalStateStore, NewLocalOptions, PrefetchOptions, ReadOptions,
+    StateStoreIterItemStream,
 };
 use risingwave_storage::table::merge_sort::merge_sort;
-use risingwave_storage::table::{compute_chunk_vnode, compute_vnode, get_second, Distribution};
+use risingwave_storage::table::{compute_chunk_vnode, compute_vnode, Distribution, KeyedRow};
 use risingwave_storage::StateStore;
 use tracing::{trace, Instrument};
 
@@ -149,6 +150,42 @@ pub type WatermarkCacheStateTable<S> =
 pub type WatermarkCacheParameterizedStateTable<S, const USE_WATERMARK_CACHE: bool> =
     StateTableInner<S, BasicSerde, false, DefaultWatermarkBufferStrategy, USE_WATERMARK_CACHE>;
 
+// initialize
+impl<S, SD, W, const USE_WATERMARK_CACHE: bool> StateTableInner<S, SD, true, W, USE_WATERMARK_CACHE>
+where
+    S: StateStore,
+    SD: ValueRowSerde,
+    W: WatermarkBufferStrategy,
+{
+    /// get the newest epoch of the state store and panic if the `init_epoch()` has never be called
+    /// async interface only used for replicated state table,
+    /// as it needs to wait for prev epoch to be committed.
+    pub async fn init_epoch(&mut self, epoch: EpochPair) -> StorageResult<()> {
+        self.local_store
+            .init(InitOptions::new_with_epoch(epoch))
+            .await
+    }
+}
+
+// initialize
+impl<S, SD, W, const USE_WATERMARK_CACHE: bool>
+    StateTableInner<S, SD, false, W, USE_WATERMARK_CACHE>
+where
+    S: StateStore,
+    SD: ValueRowSerde,
+    W: WatermarkBufferStrategy,
+{
+    /// get the newest epoch of the state store and panic if the `init_epoch()` has never be called
+    /// No need to `wait_for_epoch`, so it should complete immediately.
+    pub fn init_epoch(&mut self, epoch: EpochPair) {
+        self.local_store
+            .init(InitOptions::new_with_epoch(epoch))
+            .now_or_never()
+            .expect("non-replicated state store should start immediately.")
+            .expect("non-replicated state store should not wait_for_epoch, and fail because of it.")
+    }
+}
+
 // initialize
 impl<S, SD, const IS_REPLICATED: bool, W, const USE_WATERMARK_CACHE: bool>
     StateTableInner<S, SD, IS_REPLICATED, W, USE_WATERMARK_CACHE>
@@ -474,11 +511,6 @@ where
         }
     }
 
-    /// get the newest epoch of the state store and panic if the `init_epoch()` has never be called
-    pub fn init_epoch(&mut self, epoch: EpochPair) {
-        self.local_store.init(epoch.curr)
-    }
-
     /// get the newest epoch of the state store and panic if the `init_epoch()` has never be called
     pub fn epoch(&self) -> u64 {
         self.local_store.epoch()
@@ -883,11 +915,7 @@ where
                     let mut streams = vec![];
                     for vnode in self.vnodes().iter_vnodes() {
                         let stream = self
-                            .iter_key_and_val_with_pk_range(
-                                &range,
-                                vnode,
-                                PrefetchOptions::default(),
-                            )
+                            .iter_row_with_pk_range(&range, vnode, PrefetchOptions::default())
                             .await?;
                         streams.push(Box::pin(stream));
                     }
@@ -896,8 +924,8 @@ where
 
                     #[for_await]
                     for entry in merged_stream.take(self.watermark_cache.capacity()) {
-                        let (pk, _row) = entry?;
-                        let (_, pk) = deserialize_pk_with_vnode(&pk[..], &self.pk_serde)?;
+                        let keyed_row = entry?;
+                        let pk = self.pk_serde.deserialize(keyed_row.key())?;
                         if !pk.is_null_at(0) {
                             pks.push(pk);
                         }
@@ -1046,63 +1074,31 @@ where
     SD: ValueRowSerde,
 {
     /// This function scans rows from the relational table.
-    pub async fn iter(
+    pub async fn iter_row(
         &self,
         prefetch_options: PrefetchOptions,
-    ) -> StreamExecutorResult<RowStream<'_, S, SD>> {
-        self.iter_with_pk_prefix(row::empty(), prefetch_options)
+    ) -> StreamExecutorResult<KeyedRowStream<'_, S, SD>> {
+        self.iter_row_with_pk_prefix(row::empty(), prefetch_options)
             .await
     }
 
     /// This function scans rows from the relational table with specific `pk_prefix`.
-    pub async fn iter_with_pk_prefix(
+    /// `pk_prefix` is used to identify the exact vnode the scan should perform on.
+    pub async fn iter_row_with_pk_prefix(
         &self,
         pk_prefix: impl Row,
         prefetch_options: PrefetchOptions,
-    ) -> StreamExecutorResult<RowStream<'_, S, SD>> {
-        Ok(self
-            .iter_key_and_val(pk_prefix, prefetch_options)
-            .await?
-            .map(get_second))
-    }
-
-    /// This function scans rows from the relational table with specific `pk_prefix`.
-    async fn iter_with_pk_range_inner(
-        &self,
-        pk_range: &(Bound<impl Row>, Bound<impl Row>),
-        // Optional vnode that returns an iterator only over the given range under that vnode.
-        // For now, we require this parameter, and will panic. In the future, when `None`, we can
-        // iterate over each vnode that the `StateTableInner` owns.
-        vnode: VirtualNode,
-        prefetch_options: PrefetchOptions,
-    ) -> StreamExecutorResult<<S::Local as LocalStateStore>::IterStream<'_>> {
-        let memcomparable_range = prefix_range_to_memcomparable(&self.pk_serde, pk_range);
-
-        let memcomparable_range_with_vnode =
-            prefixed_range(memcomparable_range, &vnode.to_be_bytes());
-
-        // TODO: provide a trace of useful params.
-        self.iter_inner(memcomparable_range_with_vnode, None, prefetch_options)
-            .await
-            .map_err(StreamExecutorError::from)
-    }
-
-    pub async fn iter_with_pk_range(
-        &self,
-        pk_range: &(Bound<impl Row>, Bound<impl Row>),
-        // Optional vnode that returns an iterator only over the given range under that vnode.
-        // For now, we require this parameter, and will panic. In the future, when `None`, we can
-        // iterate over each vnode that the `StateTableInner` owns.
-        vnode: VirtualNode,
-        prefetch_options: PrefetchOptions,
-    ) -> StreamExecutorResult<RowStream<'_, S, SD>> {
-        Ok(self
-            .iter_key_and_val_with_pk_range(pk_range, vnode, prefetch_options)
-            .await?
-            .map(get_second))
+    ) -> StreamExecutorResult<KeyedRowStream<'_, S, SD>> {
+        Ok(deserialize_keyed_row_stream(
+            self.iter_kv_with_pk_prefix(pk_prefix, prefetch_options)
+                .await?,
+            &self.row_serde,
+        ))
     }
 
-    pub async fn iter_key_and_val_with_pk_range(
+    /// This function scans rows from the relational table with specific `pk_range` under the same
+    /// `vnode`.
+    pub async fn iter_row_with_pk_range(
         &self,
         pk_range: &(Bound<impl Row>, Bound<impl Row>),
         // Optional vnode that returns an iterator only over the given range under that vnode.
@@ -1110,29 +1106,36 @@ where
         // iterate over each vnode that the `StateTableInner` owns.
         vnode: VirtualNode,
         prefetch_options: PrefetchOptions,
-    ) -> StreamExecutorResult<RowStreamWithPk<'_, S, SD>> {
-        Ok(deserialize_row_stream(
-            self.iter_with_pk_range_inner(pk_range, vnode, prefetch_options)
+    ) -> StreamExecutorResult<KeyedRowStream<'_, S, SD>> {
+        Ok(deserialize_keyed_row_stream(
+            self.iter_kv_with_pk_range(pk_range, vnode, prefetch_options)
                 .await?,
             &self.row_serde,
         ))
     }
 
-    /// This function scans rows from the relational table with specific `pk_prefix`, return both
-    /// key and value.
-    pub async fn iter_key_and_val(
+    async fn iter_kv(
         &self,
-        pk_prefix: impl Row,
+        key_range: (Bound<Bytes>, Bound<Bytes>),
+        prefix_hint: Option<Bytes>,
         prefetch_options: PrefetchOptions,
-    ) -> StreamExecutorResult<RowStreamWithPk<'_, S, SD>> {
-        Ok(deserialize_row_stream(
-            self.iter_with_pk_prefix_inner(pk_prefix, prefetch_options)
-                .await?,
-            &self.row_serde,
-        ))
+    ) -> StreamExecutorResult<<S::Local as LocalStateStore>::IterStream<'_>> {
+        let read_options = ReadOptions {
+            prefix_hint,
+            ignore_range_tombstone: false,
+            retention_seconds: self.table_option.retention_seconds,
+            table_id: self.table_id,
+            read_version_from_backup: false,
+            prefetch_options,
+            cache_policy: CachePolicy::Fill(CachePriority::High),
+        };
+
+        Ok(self.local_store.iter(key_range, read_options).await?)
     }
 
-    async fn iter_with_pk_prefix_inner(
+    /// This function scans raw key-values from the relational table with specific `pk_prefix`.
+    /// `pk_prefix` is used to identify the exact vnode the scan should perform on.
+    async fn iter_kv_with_pk_prefix(
         &self,
         pk_prefix: impl Row,
         prefetch_options: PrefetchOptions,
@@ -1171,27 +1174,30 @@ where
             "storage_iter_with_prefix"
         );
 
-        self.iter_inner(encoded_key_range_with_vnode, prefix_hint, prefetch_options)
+        self.iter_kv(encoded_key_range_with_vnode, prefix_hint, prefetch_options)
             .await
     }
 
-    async fn iter_inner(
+    /// This function scans raw key-values from the relational table with specific `pk_range` under
+    /// the same `vnode`.
+    async fn iter_kv_with_pk_range(
         &self,
-        key_range: (Bound<Bytes>, Bound<Bytes>),
-        prefix_hint: Option<Bytes>,
+        pk_range: &(Bound<impl Row>, Bound<impl Row>),
+        // Optional vnode that returns an iterator only over the given range under that vnode.
+        // For now, we require this parameter, and will panic. In the future, when `None`, we can
+        // iterate over each vnode that the `StateTableInner` owns.
+        vnode: VirtualNode,
         prefetch_options: PrefetchOptions,
     ) -> StreamExecutorResult<<S::Local as LocalStateStore>::IterStream<'_>> {
-        let read_options = ReadOptions {
-            prefix_hint,
-            ignore_range_tombstone: false,
-            retention_seconds: self.table_option.retention_seconds,
-            table_id: self.table_id,
-            read_version_from_backup: false,
-            prefetch_options,
-            cache_policy: CachePolicy::Fill(CachePriority::High),
-        };
+        let memcomparable_range = prefix_range_to_memcomparable(&self.pk_serde, pk_range);
 
-        Ok(self.local_store.iter(key_range, read_options).await?)
+        let memcomparable_range_with_vnode =
+            prefixed_range(memcomparable_range, &vnode.to_be_bytes());
+
+        // TODO: provide a trace of useful params.
+        self.iter_kv(memcomparable_range_with_vnode, None, prefetch_options)
+            .await
+            .map_err(StreamExecutorError::from)
     }
 
     pub fn get_vnodes(&self) -> Arc<Bitmap> {
@@ -1250,23 +1256,21 @@ where
     }
 }
 
-pub type RowStream<'a, S: StateStore, SD: ValueRowSerde + 'a> =
-    impl Stream<Item = StreamExecutorResult<OwnedRow>> + 'a;
-pub type RowStreamWithPk<'a, S: StateStore, SD: ValueRowSerde + 'a> =
-    impl Stream<Item = StreamExecutorResult<(Bytes, OwnedRow)>> + 'a;
+pub type KeyedRowStream<'a, S: StateStore, SD: ValueRowSerde + 'a> =
+    impl Stream<Item = StreamExecutorResult<KeyedRow<Bytes>>> + 'a;
 
-fn deserialize_row_stream<'a>(
+fn deserialize_keyed_row_stream<'a>(
     stream: impl StateStoreIterItemStream + 'a,
     deserializer: &'a impl ValueRowSerde,
-) -> impl Stream<Item = StreamExecutorResult<(Bytes, OwnedRow)>> + 'a {
+) -> impl Stream<Item = StreamExecutorResult<KeyedRow<Bytes>>> + 'a {
     stream.map(move |result| {
         result
             .map_err(StreamExecutorError::from)
             .and_then(|(key, value)| {
-                Ok(deserializer
-                    .deserialize(&value)
-                    .map(OwnedRow::new)
-                    .map(move |row| (key.user_key.table_key.0, row))?)
+                Ok(KeyedRow::new(
+                    key.user_key.table_key,
+                    deserializer.deserialize(&value).map(OwnedRow::new)?,
+                ))
             })
     })
 }
diff --git a/src/stream/src/common/table/test_state_table.rs b/src/stream/src/common/table/test_state_table.rs
index 578210159fcbe..c3e5759a47ae6 100644
--- a/src/stream/src/common/table/test_state_table.rs
+++ b/src/stream/src/common/table/test_state_table.rs
@@ -281,7 +281,7 @@ async fn test_state_table_iter_with_prefix() {
 
     let pk_prefix = OwnedRow::new(vec![Some(1_i32.into())]);
     let iter = state_table
-        .iter_with_pk_prefix(&pk_prefix, Default::default())
+        .iter_row_with_pk_prefix(&pk_prefix, Default::default())
         .await
         .unwrap();
     pin_mut!(iter);
@@ -412,7 +412,7 @@ async fn test_state_table_iter_with_pk_range() {
         std::ops::Bound::Included(OwnedRow::new(vec![Some(4_i32.into())])),
     );
     let iter = state_table
-        .iter_with_pk_range(&pk_range, DEFAULT_VNODE, Default::default())
+        .iter_row_with_pk_range(&pk_range, DEFAULT_VNODE, Default::default())
         .await
         .unwrap();
     pin_mut!(iter);
@@ -437,7 +437,7 @@ async fn test_state_table_iter_with_pk_range() {
         std::ops::Bound::<row::Empty>::Unbounded,
     );
     let iter = state_table
-        .iter_with_pk_range(&pk_range, DEFAULT_VNODE, Default::default())
+        .iter_row_with_pk_range(&pk_range, DEFAULT_VNODE, Default::default())
         .await
         .unwrap();
     pin_mut!(iter);
@@ -576,7 +576,7 @@ async fn test_state_table_iter_with_value_indices() {
     ]));
 
     {
-        let iter = state_table.iter(Default::default()).await.unwrap();
+        let iter = state_table.iter_row(Default::default()).await.unwrap();
         pin_mut!(iter);
 
         let res = iter.next().await.unwrap().unwrap();
@@ -631,7 +631,7 @@ async fn test_state_table_iter_with_value_indices() {
         Some(888_i32.into()),
     ]));
 
-    let iter = state_table.iter(Default::default()).await.unwrap();
+    let iter = state_table.iter_row(Default::default()).await.unwrap();
     pin_mut!(iter);
 
     let res = iter.next().await.unwrap().unwrap();
@@ -737,7 +737,7 @@ async fn test_state_table_iter_with_shuffle_value_indices() {
     ]));
 
     {
-        let iter = state_table.iter(Default::default()).await.unwrap();
+        let iter = state_table.iter_row(Default::default()).await.unwrap();
         pin_mut!(iter);
 
         let res = iter.next().await.unwrap().unwrap();
@@ -813,7 +813,7 @@ async fn test_state_table_iter_with_shuffle_value_indices() {
         Some(888_i32.into()),
     ]));
 
-    let iter = state_table.iter(Default::default()).await.unwrap();
+    let iter = state_table.iter_row(Default::default()).await.unwrap();
     pin_mut!(iter);
 
     let res = iter.next().await.unwrap().unwrap();
@@ -1000,13 +1000,13 @@ async fn test_state_table_write_chunk() {
     state_table.write_chunk(chunk);
 
     let rows: Vec<_> = state_table
-        .iter(PrefetchOptions::new_for_exhaust_iter())
+        .iter_row(PrefetchOptions::new_for_exhaust_iter())
         .await
         .unwrap()
         .collect::<Vec<_>>()
         .await
         .into_iter()
-        .map(|row| row.unwrap())
+        .map(|row| row.unwrap().into_owned_row())
         .collect();
 
     assert_eq!(rows.len(), 2);
@@ -1117,13 +1117,13 @@ async fn test_state_table_write_chunk_visibility() {
     state_table.write_chunk(chunk);
 
     let rows: Vec<_> = state_table
-        .iter(PrefetchOptions::new_for_exhaust_iter())
+        .iter_row(PrefetchOptions::new_for_exhaust_iter())
         .await
         .unwrap()
         .collect::<Vec<_>>()
         .await
         .into_iter()
-        .map(|row| row.unwrap())
+        .map(|row| row.unwrap().into_owned_row())
         .collect();
 
     assert_eq!(rows.len(), 3);
@@ -1229,13 +1229,13 @@ async fn test_state_table_write_chunk_value_indices() {
     state_table.write_chunk(chunk);
 
     let rows: Vec<_> = state_table
-        .iter(PrefetchOptions::new_for_exhaust_iter())
+        .iter_row(PrefetchOptions::new_for_exhaust_iter())
         .await
         .unwrap()
         .collect::<Vec<_>>()
         .await
         .into_iter()
-        .map(|row| row.unwrap())
+        .map(|row| row.unwrap().into_owned_row())
         .collect();
 
     assert_eq!(rows.len(), 3);
@@ -1511,13 +1511,13 @@ async fn test_state_table_watermark_cache_ignore_null() {
     state_table.write_chunk(chunk);
 
     let inserted_rows: Vec<_> = state_table
-        .iter(PrefetchOptions::new_for_exhaust_iter())
+        .iter_row(PrefetchOptions::new_for_exhaust_iter())
         .await
         .unwrap()
         .collect::<Vec<_>>()
         .await
         .into_iter()
-        .map(|row| row.unwrap())
+        .map(|row| row.unwrap().into_owned_row())
         .collect();
 
     assert_eq!(inserted_rows.len(), 4);
@@ -1798,13 +1798,13 @@ async fn test_state_table_watermark_cache_refill() {
     }
 
     let inserted_rows: Vec<_> = state_table
-        .iter(PrefetchOptions::new_for_exhaust_iter())
+        .iter_row(PrefetchOptions::new_for_exhaust_iter())
         .await
         .unwrap()
         .collect::<Vec<_>>()
         .await
         .into_iter()
-        .map(|row| row.unwrap())
+        .map(|row| row.unwrap().into_owned_row())
         .collect();
 
     assert_eq!(inserted_rows.len(), 4);
diff --git a/src/stream/src/error.rs b/src/stream/src/error.rs
index 5c2ebf030cb0f..69cb51ca0d287 100644
--- a/src/stream/src/error.rs
+++ b/src/stream/src/error.rs
@@ -16,6 +16,7 @@ use std::backtrace::Backtrace;
 use std::error::request_ref;
 
 use risingwave_common::array::ArrayError;
+use risingwave_connector::error::ConnectorError;
 use risingwave_expr::ExprError;
 use risingwave_pb::PbFieldNotFound;
 use risingwave_storage::error::StorageError;
@@ -122,6 +123,12 @@ impl From<PbFieldNotFound> for StreamError {
     }
 }
 
+impl From<ConnectorError> for StreamError {
+    fn from(err: ConnectorError) -> Self {
+        StreamExecutorError::from(err).into()
+    }
+}
+
 // Internal error.
 impl From<anyhow::Error> for StreamError {
     fn from(a: anyhow::Error) -> Self {
diff --git a/src/stream/src/executor/aggregation/agg_group.rs b/src/stream/src/executor/aggregation/agg_group.rs
index d875b6c104cb2..fb93f600e1fb6 100644
--- a/src/stream/src/executor/aggregation/agg_group.rs
+++ b/src/stream/src/executor/aggregation/agg_group.rs
@@ -17,19 +17,17 @@ use std::marker::PhantomData;
 use std::sync::Arc;
 
 use risingwave_common::array::stream_record::{Record, RecordType};
-use risingwave_common::array::{Op, StreamChunk, Vis};
+use risingwave_common::array::{StreamChunk, Vis};
 use risingwave_common::catalog::Schema;
 use risingwave_common::estimate_size::EstimateSize;
 use risingwave_common::must_match;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
-use risingwave_common::types::DataType;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_expr::agg::{AggCall, BoxedAggregateFunction};
 use risingwave_storage::StateStore;
 
 use super::agg_state::{AggState, AggStateStorage};
 use crate::common::table::state_table::StateTable;
-use crate::common::StreamChunkBuilder;
 use crate::executor::error::StreamExecutorResult;
 use crate::executor::PkIndices;
 
@@ -408,45 +406,3 @@ impl<S: StateStore, Strtg: Strategy> AggGroup<S, Strtg> {
         }))
     }
 }
-
-// TODO(rc): split logic of `StreamChunkBuilder` to chunk builder and row merger.
-/// A wrapper of [`StreamChunkBuilder`] that provides a more convenient API.
-pub struct ChunkBuilder {
-    inner: StreamChunkBuilder,
-}
-
-impl ChunkBuilder {
-    pub fn new(capacity: usize, data_types: &[DataType]) -> Self {
-        Self {
-            inner: StreamChunkBuilder::new(
-                capacity,
-                data_types,
-                (0..data_types.len()).map(|x| (x, x)).collect(),
-                vec![],
-            ),
-        }
-    }
-
-    /// Append a row to the builder, return a chunk if the builder is full.
-    #[must_use]
-    pub fn append_row(&mut self, op: Op, row: impl Row) -> Option<StreamChunk> {
-        self.inner.append_row_update(op, row)
-    }
-
-    /// Append a record to the builder, return a chunk if the builder is full.
-    pub fn append_record(&mut self, record: Record<impl Row>) -> Option<StreamChunk> {
-        match record {
-            Record::Insert { new_row } => self.append_row(Op::Insert, new_row),
-            Record::Delete { old_row } => self.append_row(Op::Delete, old_row),
-            Record::Update { old_row, new_row } => {
-                let _none = self.append_row(Op::UpdateDelete, old_row);
-                self.append_row(Op::UpdateInsert, new_row)
-            }
-        }
-    }
-
-    /// Take remaining rows and build a chunk.
-    pub fn take(&mut self) -> Option<StreamChunk> {
-        self.inner.take()
-    }
-}
diff --git a/src/stream/src/executor/aggregation/minput.rs b/src/stream/src/executor/aggregation/minput.rs
index c6123b5317604..7d65e9f787938 100644
--- a/src/stream/src/executor/aggregation/minput.rs
+++ b/src/stream/src/executor/aggregation/minput.rs
@@ -18,7 +18,7 @@ use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::catalog::Schema;
 use risingwave_common::estimate_size::EstimateSize;
-use risingwave_common::row::{OwnedRow, RowExt};
+use risingwave_common::row::RowExt;
 use risingwave_common::types::Datum;
 use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common::util::sort_util::OrderType;
@@ -184,7 +184,7 @@ impl MaterializedInputState {
             let mut cache_filler = self.cache.begin_syncing();
 
             let all_data_iter = state_table
-                .iter_with_pk_prefix(
+                .iter_row_with_pk_prefix(
                     group_key.map(GroupKey::table_pk),
                     PrefetchOptions {
                         exhaust_iter: cache_filler.capacity().is_none(),
@@ -194,8 +194,8 @@ impl MaterializedInputState {
             pin_mut!(all_data_iter);
 
             #[for_await]
-            for state_row in all_data_iter.take(cache_filler.capacity().unwrap_or(usize::MAX)) {
-                let state_row: OwnedRow = state_row?;
+            for keyed_row in all_data_iter.take(cache_filler.capacity().unwrap_or(usize::MAX)) {
+                let state_row = keyed_row?;
                 let cache_key = {
                     let mut cache_key = Vec::new();
                     self.cache_key_serializer.serialize(
diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs
index 859b1723fad24..d33aed6d6c441 100644
--- a/src/stream/src/executor/backfill/arrangement_backfill.rs
+++ b/src/stream/src/executor/backfill/arrangement_backfill.rs
@@ -34,8 +34,8 @@ use risingwave_storage::StateStore;
 use crate::common::table::state_table::ReplicatedStateTable;
 use crate::executor::backfill::utils::{
     compute_bounds, construct_initial_finished_state, get_progress_per_vnode, iter_chunks,
-    mapping_chunk, mapping_message, mark_chunk_ref_by_vnode, persist_state_per_vnode,
-    update_pos_by_vnode, BackfillProgressPerVnode, BackfillState,
+    mapping_chunk, mapping_message, mark_chunk_ref_by_vnode, owned_row_iter,
+    persist_state_per_vnode, update_pos_by_vnode, BackfillProgressPerVnode, BackfillState,
 };
 use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{
@@ -44,7 +44,7 @@ use crate::executor::{
 };
 use crate::task::{ActorId, CreateMviewProgress};
 
-/// Similar to [`BackfillExecutor`].
+/// Similar to [`super::no_shuffle_backfill::BackfillExecutor`].
 /// Main differences:
 /// - [`ArrangementBackfillExecutor`] can reside on a different CN, so it can be scaled
 ///   independently.
@@ -130,7 +130,7 @@ where
 
         // Poll the upstream to get the first barrier.
         let first_barrier = expect_first_barrier(&mut upstream).await?;
-        self.state_table.init_epoch(first_barrier.epoch);
+        self.state_table.init_epoch(first_barrier.epoch).await?;
 
         let progress_per_vnode = get_progress_per_vnode(&self.state_table).await?;
 
@@ -162,7 +162,6 @@ where
                 let snapshot = Self::snapshot_read_per_vnode(
                     &upstream_table,
                     backfill_state.clone(), // FIXME: temporary workaround... How to avoid it?
-                    self.chunk_size,
                     &mut builders,
                 );
                 pin_mut!(snapshot);
@@ -244,7 +243,6 @@ where
                     let right_snapshot = pin!(Self::snapshot_read_per_vnode(
                         &upstream_table,
                         backfill_state.clone(), // FIXME: temporary workaround, how to avoid it?
-                        self.chunk_size,
                         &mut builders,
                     )
                     .map(Either::Right),);
@@ -348,19 +346,27 @@ where
                 // NOTE(kwannoel): `zip_eq_debug` does not work here,
                 // we encounter "higher-ranked lifetime error".
                 for (vnode, chunk) in vnodes.iter_vnodes().zip_eq(builders.iter_mut().map(|b| {
-                    let chunk = b.build_data_chunk();
-                    let ops = vec![Op::Insert; chunk.capacity()];
-                    StreamChunk::from_parts(ops, chunk)
+                    b.consume_all().map(|chunk| {
+                        let ops = vec![Op::Insert; chunk.capacity()];
+                        StreamChunk::from_parts(ops, chunk)
+                    })
                 })) {
-                    // Raise the current position.
-                    // As snapshot read streams are ordered by pk, so we can
-                    // just use the last row to update `current_pos`.
-                    update_pos_by_vnode(vnode, &chunk, &pk_in_output_indices, &mut backfill_state);
-
-                    let chunk_cardinality = chunk.cardinality() as u64;
-                    cur_barrier_snapshot_processed_rows += chunk_cardinality;
-                    total_snapshot_processed_rows += chunk_cardinality;
-                    yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
+                    if let Some(chunk) = chunk {
+                        // Raise the current position.
+                        // As snapshot read streams are ordered by pk, so we can
+                        // just use the last row to update `current_pos`.
+                        update_pos_by_vnode(
+                            vnode,
+                            &chunk,
+                            &pk_in_output_indices,
+                            &mut backfill_state,
+                        );
+
+                        let chunk_cardinality = chunk.cardinality() as u64;
+                        cur_barrier_snapshot_processed_rows += chunk_cardinality;
+                        total_snapshot_processed_rows += chunk_cardinality;
+                        yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
+                    }
                 }
 
                 // consume upstream buffer chunk
@@ -496,10 +502,6 @@ where
     /// 3. Change it into a chunk iterator with `iter_chunks`.
     /// This means it should fetch a row from each iterator to form a chunk.
     ///
-    /// NOTE(kwannoel): We interleave at chunk per vnode level rather than rows.
-    /// This is so that we can compute `current_pos` once per chunk, since they correspond to 1
-    /// vnode.
-    ///
     /// We will return chunks based on the `BackfillProgressPerVnode`.
     /// 1. Completed(vnode): Current iterator is complete, in that case we need to handle it
     ///    in arrangement backfill. We should not buffer updates for this vnode,
@@ -507,11 +509,21 @@ where
     /// 2. InProgress(CHUNK): Current iterator is not complete, in that case we
     ///    need to buffer updates for this vnode.
     /// 3. Finished: All iterators finished.
+    ///
+    /// NOTE(kwannoel): We interleave at chunk per vnode level rather than rows.
+    /// This is so that we can compute `current_pos` once per chunk, since they correspond to 1
+    /// vnode.
+    ///
+    /// NOTE(kwannoel):
+    /// The rows from upstream snapshot read will be buffered inside the `builder`.
+    /// If snapshot is dropped before its rows are consumed,
+    /// remaining data in `builder` must be flushed manually.
+    /// Otherwise when we scan a new snapshot, it is possible the rows in the `builder` would be
+    /// present, Then when we flush we contain duplicate rows.
     #[try_stream(ok = Option<(VirtualNode, StreamChunk)>, error = StreamExecutorError)]
     async fn snapshot_read_per_vnode<'a>(
         upstream_table: &'a ReplicatedStateTable<S>,
         backfill_state: BackfillState,
-        chunk_size: usize,
         builders: &'a mut [DataChunkBuilder],
     ) {
         let mut streams = Vec::with_capacity(upstream_table.vnodes().len());
@@ -536,13 +548,13 @@ where
             let range_bounds = range_bounds.unwrap();
 
             let vnode_row_iter = upstream_table
-                .iter_with_pk_range(&range_bounds, vnode, Default::default())
+                .iter_row_with_pk_range(&range_bounds, vnode, Default::default())
                 .await?;
 
             // TODO: Is there some way to avoid double-pin here?
-            let vnode_row_iter = Box::pin(vnode_row_iter);
+            let vnode_row_iter = Box::pin(owned_row_iter(vnode_row_iter));
 
-            let vnode_chunk_iter = iter_chunks(vnode_row_iter, chunk_size, builder)
+            let vnode_chunk_iter = iter_chunks(vnode_row_iter, builder)
                 .map_ok(move |chunk_opt| chunk_opt.map(|chunk| (vnode, chunk)));
             // TODO: Is there some way to avoid double-pin
             streams.push(Box::pin(vnode_chunk_iter));
diff --git a/src/stream/src/executor/backfill/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc_backfill.rs
new file mode 100644
index 0000000000000..2f522ae8eeb0c
--- /dev/null
+++ b/src/stream/src/executor/backfill/cdc_backfill.rs
@@ -0,0 +1,563 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::default::Default;
+use std::pin::{pin, Pin};
+use std::sync::Arc;
+
+use anyhow::anyhow;
+use either::Either;
+use futures::stream::select_with_strategy;
+use futures::{pin_mut, stream, StreamExt, TryStreamExt};
+use futures_async_stream::try_stream;
+use itertools::Itertools;
+use maplit::hashmap;
+use risingwave_common::array::StreamChunk;
+use risingwave_common::catalog::Schema;
+use risingwave_common::row::{OwnedRow, Row};
+use risingwave_common::types::{JsonbVal, ScalarRefImpl};
+use risingwave_common::util::epoch::EpochPair;
+use risingwave_connector::source::external::{CdcOffset, DebeziumOffset, DebeziumSourceOffset};
+use risingwave_connector::source::{SplitId, SplitImpl, SplitMetaData};
+use risingwave_storage::StateStore;
+use serde_json::Value;
+
+use crate::executor::backfill::upstream_table::external::ExternalStorageTable;
+use crate::executor::backfill::upstream_table::snapshot::{
+    SnapshotReadArgs, UpstreamTableRead, UpstreamTableReader,
+};
+use crate::executor::backfill::utils::{
+    get_cdc_chunk_last_offset, get_new_pos, mapping_chunk, mapping_message, mark_cdc_chunk,
+};
+use crate::executor::monitor::StreamingMetrics;
+use crate::executor::{
+    expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor,
+    ExecutorInfo, Message, Mutation, PkIndices, PkIndicesRef, SourceStateTableHandler,
+    StreamExecutorError, StreamExecutorResult,
+};
+use crate::task::{ActorId, CreateMviewProgress};
+
+const BACKFILL_STATE_KEY_SUFFIX: &str = "_backfill";
+
+pub struct CdcBackfillExecutor<S: StateStore> {
+    actor_ctx: ActorContextRef,
+
+    /// Upstream external table
+    upstream_table: ExternalStorageTable,
+
+    /// Upstream changelog with the same schema with the external table.
+    upstream: BoxedExecutor,
+
+    /// The column indices need to be forwarded to the downstream from the upstream and table scan.
+    /// User may select a subset of columns from the upstream table.
+    output_indices: Vec<usize>,
+
+    actor_id: ActorId,
+
+    info: ExecutorInfo,
+
+    /// Stores the backfill done flag
+    source_state_handler: SourceStateTableHandler<S>,
+
+    metrics: Arc<StreamingMetrics>,
+
+    chunk_size: usize,
+}
+
+impl<S: StateStore> CdcBackfillExecutor<S> {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        actor_ctx: ActorContextRef,
+        upstream_table: ExternalStorageTable,
+        upstream: BoxedExecutor,
+        output_indices: Vec<usize>,
+        _progress: Option<CreateMviewProgress>,
+        schema: Schema,
+        pk_indices: PkIndices,
+        metrics: Arc<StreamingMetrics>,
+        source_state_handler: SourceStateTableHandler<S>,
+        chunk_size: usize,
+    ) -> Self {
+        Self {
+            info: ExecutorInfo {
+                schema,
+                pk_indices,
+                identity: "CdcBackfillExecutor".to_owned(),
+            },
+            upstream_table,
+            upstream,
+            output_indices,
+            actor_id: 0,
+            metrics,
+            chunk_size,
+            actor_ctx,
+            source_state_handler,
+        }
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn execute_inner(mut self) {
+        // The primary key columns, in the output columns of the upstream_table scan.
+        let pk_in_output_indices = self.upstream_table.pk_in_output_indices().unwrap();
+        let pk_order = self.upstream_table.pk_order_types().to_vec();
+
+        let upstream_table_id = self.upstream_table.table_id().table_id;
+        let upstream_table_reader = UpstreamTableReader::new(self.upstream_table);
+
+        let mut upstream = self.upstream.execute().peekable();
+
+        // Current position of the upstream_table storage primary key.
+        // `None` means it starts from the beginning.
+        let mut current_pk_pos: Option<OwnedRow>;
+
+        tracing::info!(upstream_table_id, ?pk_in_output_indices);
+
+        // Poll the upstream to get the first barrier.
+        let first_barrier = expect_first_barrier(&mut upstream).await?;
+        let init_epoch = first_barrier.epoch.prev;
+
+        // Check whether this parallelism has been assigned splits,
+        // if not, we should bypass the backfill directly.
+        let mut invalid_backfill = false;
+        let mut split_id: Option<SplitId> = None;
+        let mut cdc_split: Option<SplitImpl> = None;
+        if let Some(mutation) = first_barrier.mutation.as_ref() {
+            match mutation.as_ref() {
+                Mutation::Add { splits, .. }
+                | Mutation::Update {
+                    actor_splits: splits,
+                    ..
+                } => {
+                    invalid_backfill = match splits.get(&self.actor_ctx.id) {
+                        None => true,
+                        Some(splits) => {
+                            let is_empty = splits.is_empty();
+                            if !is_empty {
+                                let split = splits.iter().exactly_one().map_err(|err| {
+                                    StreamExecutorError::from(anyhow!(
+                                        "fail to get cdc split {}",
+                                        err
+                                    ))
+                                })?;
+                                split_id = Some(split.id());
+                                cdc_split = Some(split.clone());
+                            }
+                            is_empty
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        if invalid_backfill {
+            // The first barrier message should be propagated.
+            yield Message::Barrier(first_barrier);
+            #[for_await]
+            for msg in upstream {
+                if let Some(msg) = mapping_message(msg?, &self.output_indices) {
+                    yield msg;
+                }
+            }
+            // exit the executor
+            return Ok(());
+        }
+
+        tracing::debug!("start cdc backfill: actor {:?}", self.actor_ctx.id);
+
+        self.source_state_handler.init_epoch(first_barrier.epoch);
+
+        // start from the beginning
+        // TODO(siyuan): restore backfill offset from state store
+        let backfill_offset = None;
+
+        current_pk_pos = backfill_offset;
+
+        let is_finished = {
+            // restore backfill done flag from state store
+            if let Some(split_id) = split_id.as_ref() {
+                let mut key = split_id.to_string();
+                key.push_str(BACKFILL_STATE_KEY_SUFFIX);
+                match self.source_state_handler.get(key.into()).await? {
+                    Some(row) => match row.datum_at(1) {
+                        Some(ScalarRefImpl::Jsonb(jsonb_ref)) => jsonb_ref.as_bool()?,
+                        _ => unreachable!("invalid backfill persistent state"),
+                    },
+                    None => false,
+                }
+            } else {
+                false
+            }
+        };
+
+        // If the snapshot is empty, we don't need to backfill.
+        let is_snapshot_empty: bool = {
+            if is_finished {
+                // It is finished, so just assign a value to avoid accessing storage table again.
+                false
+            } else {
+                let args = SnapshotReadArgs::new(init_epoch, None, false, self.chunk_size);
+                let snapshot = upstream_table_reader.snapshot_read(args);
+                pin_mut!(snapshot);
+                snapshot.try_next().await?.unwrap().is_none()
+            }
+        };
+
+        // | backfill_is_finished | snapshot_empty | need_to_backfill |
+        // | t                    | t/f            | f                |
+        // | f                    | t              | f                |
+        // | f                    | f              | t                |
+        let to_backfill = !is_finished && !is_snapshot_empty;
+
+        // The first barrier message should be propagated.
+        yield Message::Barrier(first_barrier);
+
+        // Keep track of rows from the snapshot.
+        #[allow(unused_variables)]
+        let mut total_snapshot_processed_rows: u64 = 0;
+
+        let mut last_binlog_offset: Option<CdcOffset>;
+
+        let mut consumed_binlog_offset: Option<CdcOffset> = None;
+
+        // CDC Backfill Algorithm:
+        //
+        // When the first barrier comes from upstream:
+        //  - read the current binlog offset as `binlog_low`
+        //  - start a snapshot read upon upstream table and iterate over the snapshot read stream
+        //  - buffer the changelog event from upstream
+        //
+        // When a new barrier comes from upstream:
+        //  - read the current binlog offset as `binlog_high`
+        //  - for each row of the upstream change log, forward it to downstream if it in the range
+        //    of [binlog_low, binlog_high] and its pk <= `current_pos`, otherwise ignore it
+        //  - reconstruct the whole backfill stream with upstream changelog and a new table snapshot
+        //
+        // When a chunk comes from snapshot, we forward it to the downstream and raise
+        // `current_pos`.
+        // When we reach the end of the snapshot read stream, it means backfill has been
+        // finished.
+        //
+        // Once the backfill loop ends, we forward the upstream directly to the downstream.
+        if to_backfill {
+            last_binlog_offset = upstream_table_reader.current_binlog_offset().await?;
+            // drive the upstream changelog first to ensure we can receive timely changelog event,
+            // otherwise the upstream changelog may be blocked by the snapshot read stream
+            let _ = Pin::new(&mut upstream).peek().await;
+
+            tracing::info!(
+                upstream_table_id,
+                initial_binlog_offset = ?last_binlog_offset,
+                "start the bacfill loop");
+
+            'backfill_loop: loop {
+                let mut upstream_chunk_buffer: Vec<StreamChunk> = vec![];
+
+                let left_upstream = upstream.by_ref().map(Either::Left);
+
+                let args = SnapshotReadArgs::new_for_cdc(current_pk_pos.clone(), self.chunk_size);
+                let right_snapshot =
+                    pin!(upstream_table_reader.snapshot_read(args).map(Either::Right));
+
+                // Prefer to select upstream, so we can stop snapshot stream when barrier comes.
+                let backfill_stream =
+                    select_with_strategy(left_upstream, right_snapshot, |_: &mut ()| {
+                        stream::PollNext::Left
+                    });
+
+                let mut cur_barrier_snapshot_processed_rows: u64 = 0;
+                let mut cur_barrier_upstream_processed_rows: u64 = 0;
+
+                #[for_await]
+                for either in backfill_stream {
+                    match either {
+                        // Upstream
+                        Either::Left(msg) => {
+                            match msg? {
+                                Message::Barrier(barrier) => {
+                                    // If it is a barrier, switch snapshot and consume buffered
+                                    // upstream chunk.
+                                    // If no current_pos, means we did not process any snapshot yet.
+                                    // In that case we can just ignore the upstream buffer chunk.
+                                    if let Some(current_pos) = &current_pk_pos {
+                                        for chunk in upstream_chunk_buffer.drain(..) {
+                                            cur_barrier_upstream_processed_rows +=
+                                                chunk.cardinality() as u64;
+
+                                            // record the consumed binlog offset that will be
+                                            // persisted later
+                                            consumed_binlog_offset = get_cdc_chunk_last_offset(
+                                                upstream_table_reader.inner().table_reader(),
+                                                &chunk,
+                                            )?;
+                                            yield Message::Chunk(mapping_chunk(
+                                                mark_cdc_chunk(
+                                                    upstream_table_reader.inner().table_reader(),
+                                                    chunk,
+                                                    current_pos,
+                                                    &pk_in_output_indices,
+                                                    &pk_order,
+                                                    last_binlog_offset.clone(),
+                                                )?,
+                                                &self.output_indices,
+                                            ));
+                                        }
+                                    }
+
+                                    self.metrics
+                                        .backfill_snapshot_read_row_count
+                                        .with_label_values(&[
+                                            upstream_table_id.to_string().as_str(),
+                                            self.actor_id.to_string().as_str(),
+                                        ])
+                                        .inc_by(cur_barrier_snapshot_processed_rows);
+
+                                    self.metrics
+                                        .backfill_upstream_output_row_count
+                                        .with_label_values(&[
+                                            upstream_table_id.to_string().as_str(),
+                                            self.actor_id.to_string().as_str(),
+                                        ])
+                                        .inc_by(cur_barrier_upstream_processed_rows);
+
+                                    // Update last seen binlog offset
+                                    if consumed_binlog_offset.is_some() {
+                                        last_binlog_offset = consumed_binlog_offset.clone();
+                                    }
+
+                                    // seal current epoch even though there is no data
+                                    Self::persist_state(
+                                        &mut self.source_state_handler,
+                                        barrier.epoch,
+                                    )
+                                    .await?;
+
+                                    yield Message::Barrier(barrier);
+                                    // Break the for loop and start a new snapshot read stream.
+                                    break;
+                                }
+                                Message::Chunk(chunk) => {
+                                    let chunk_binlog_offset = get_cdc_chunk_last_offset(
+                                        upstream_table_reader.inner().table_reader(),
+                                        &chunk,
+                                    )?;
+
+                                    tracing::trace!(
+                                        "recv changelog chunk: bin offset {:?}, capactiy {}",
+                                        chunk_binlog_offset,
+                                        chunk.capacity()
+                                    );
+
+                                    // Since we don't need changelog before the
+                                    // `last_binlog_offset`, skip the chunk that *only* contains
+                                    // events before `last_binlog_offset`.
+                                    if let Some(last_binlog_offset) = &last_binlog_offset {
+                                        if let Some(chunk_binlog_offset) = chunk_binlog_offset {
+                                            if chunk_binlog_offset < *last_binlog_offset {
+                                                tracing::trace!(
+                                                    "skip changelog chunk: offset {:?}, capacity {}",
+                                                    chunk_binlog_offset,
+                                                    chunk.capacity()
+                                                );
+                                                continue;
+                                            }
+                                        }
+                                    }
+                                    // Buffer the upstream chunk.
+                                    upstream_chunk_buffer.push(chunk.compact());
+                                }
+                                Message::Watermark(_) => {
+                                    // Ignore watermark during backfill.
+                                }
+                            }
+                        }
+                        // Snapshot read
+                        Either::Right(msg) => {
+                            match msg? {
+                                None => {
+                                    tracing::info!(
+                                        upstream_table_id,
+                                        ?last_binlog_offset,
+                                        ?current_pk_pos,
+                                        "snapshot read stream ends"
+                                    );
+                                    // End of the snapshot read stream.
+                                    // We should not mark the chunk anymore,
+                                    // otherwise, we will ignore some rows
+                                    // in the buffer. Here we choose to never mark the chunk.
+                                    // Consume with the renaming stream buffer chunk without mark.
+                                    for chunk in upstream_chunk_buffer.drain(..) {
+                                        let chunk_cardinality = chunk.cardinality() as u64;
+                                        cur_barrier_snapshot_processed_rows += chunk_cardinality;
+                                        total_snapshot_processed_rows += chunk_cardinality;
+                                        yield Message::Chunk(mapping_chunk(
+                                            chunk,
+                                            &self.output_indices,
+                                        ));
+                                    }
+
+                                    Self::write_backfill_state(
+                                        &mut self.source_state_handler,
+                                        upstream_table_id,
+                                        &split_id,
+                                        &mut cdc_split,
+                                        last_binlog_offset.clone(),
+                                    )
+                                    .await?;
+                                    break 'backfill_loop;
+                                }
+                                Some(chunk) => {
+                                    // Raise the current position.
+                                    // As snapshot read streams are ordered by pk, so we can
+                                    // just use the last row to update `current_pos`.
+                                    current_pk_pos =
+                                        Some(get_new_pos(&chunk, &pk_in_output_indices));
+
+                                    tracing::trace!(
+                                        "current backfill progress: {:?}",
+                                        current_pk_pos
+                                    );
+                                    let chunk_cardinality = chunk.cardinality() as u64;
+                                    cur_barrier_snapshot_processed_rows += chunk_cardinality;
+                                    total_snapshot_processed_rows += chunk_cardinality;
+                                    yield Message::Chunk(mapping_chunk(
+                                        chunk,
+                                        &self.output_indices,
+                                    ));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } else {
+            Self::write_backfill_state(
+                &mut self.source_state_handler,
+                upstream_table_id,
+                &split_id,
+                &mut cdc_split,
+                None,
+            )
+            .await?;
+        }
+
+        tracing::debug!(
+            actor = self.actor_id,
+            "CdcBackfill has already finished and forward messages directly to the downstream"
+        );
+
+        // After backfill progress finished
+        // we can forward messages directly to the downstream,
+        // as backfill is finished.
+        #[for_await]
+        for msg in upstream {
+            // upstream offsets will be removed from the message before forwarding to
+            // downstream
+            if let Some(msg) = mapping_message(msg?, &self.output_indices) {
+                // persist the backfill state if any
+                if let Message::Barrier(barrier) = &msg {
+                    Self::persist_state(&mut self.source_state_handler, barrier.epoch).await?;
+                }
+                yield msg;
+            }
+        }
+    }
+
+    /// When snapshot read stream ends, we should persist two states:
+    /// 1) a backfill finish flag to denote the backfill has done
+    /// 2) a consumed binlog offset to denote the last binlog offset
+    /// which will be committed to the state store upon next barrier.
+    async fn write_backfill_state(
+        source_state_handler: &mut SourceStateTableHandler<S>,
+        upstream_table_id: u32,
+        split_id: &Option<SplitId>,
+        cdc_split: &mut Option<SplitImpl>,
+        last_binlog_offset: Option<CdcOffset>,
+    ) -> StreamExecutorResult<()> {
+        if let Some(split_id) = split_id.as_ref() {
+            let mut key = split_id.to_string();
+            key.push_str(BACKFILL_STATE_KEY_SUFFIX);
+            source_state_handler
+                .set(key.into(), JsonbVal::from(Value::Bool(true)))
+                .await?;
+
+            if let Some(SplitImpl::MysqlCdc(split)) = cdc_split.as_mut()
+                && let Some(s) = split.mysql_split.as_mut() {
+                let start_offset =
+                    last_binlog_offset.as_ref().map(|cdc_offset| {
+                        let source_offset =
+                            if let CdcOffset::MySql(o) = cdc_offset
+                            {
+                                DebeziumSourceOffset {
+                                    file: Some(o.filename.clone()),
+                                    pos: Some(o.position),
+                                    ..Default::default()
+                                }
+                            } else {
+                                DebeziumSourceOffset::default()
+                            };
+
+                        let mut server = "RW_CDC_".to_string();
+                        server.push_str(
+                            upstream_table_id.to_string().as_str(),
+                        );
+                        DebeziumOffset {
+                            source_partition: hashmap! {
+                                "server".to_string() => server
+                            },
+                            source_offset,
+                        }
+                    });
+
+                // persist the last binlog offset into split state
+                s.inner.start_offset = start_offset.map(|o| {
+                    let value = serde_json::to_value(o).unwrap();
+                    value.to_string()
+                });
+                s.inner.snapshot_done = true;
+            }
+            if let Some(split_impl) = cdc_split {
+                source_state_handler
+                    .set(split_impl.id(), split_impl.encode_to_json())
+                    .await?
+            }
+        }
+        Ok(())
+    }
+
+    async fn persist_state(
+        source_state_handler: &mut SourceStateTableHandler<S>,
+        new_epoch: EpochPair,
+    ) -> StreamExecutorResult<()> {
+        source_state_handler.state_store.commit(new_epoch).await
+    }
+}
+
+impl<S: StateStore> Executor for CdcBackfillExecutor<S> {
+    fn execute(self: Box<Self>) -> BoxedMessageStream {
+        self.execute_inner().boxed()
+    }
+
+    fn schema(&self) -> &Schema {
+        &self.info.schema
+    }
+
+    fn pk_indices(&self) -> PkIndicesRef<'_> {
+        &self.info.pk_indices
+    }
+
+    fn identity(&self) -> &str {
+        &self.info.identity
+    }
+}
diff --git a/src/stream/src/executor/backfill/mod.rs b/src/stream/src/executor/backfill/mod.rs
index 982c98fcd61ff..0f8e3373fd247 100644
--- a/src/stream/src/executor/backfill/mod.rs
+++ b/src/stream/src/executor/backfill/mod.rs
@@ -13,5 +13,7 @@
 // limitations under the License.
 
 pub mod arrangement_backfill;
+pub mod cdc_backfill;
 pub mod no_shuffle_backfill;
+pub mod upstream_table;
 pub mod utils;
diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs
index 344b16580800c..6040a6745f9e3 100644
--- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs
+++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs
@@ -29,14 +29,13 @@ use risingwave_common::{bail, row};
 use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::table::batch_table::storage_table::StorageTable;
-use risingwave_storage::table::get_second;
 use risingwave_storage::StateStore;
 
 use crate::common::table::state_table::StateTable;
 use crate::executor::backfill::utils;
 use crate::executor::backfill::utils::{
     check_all_vnode_finished, compute_bounds, construct_initial_finished_state, get_new_pos,
-    iter_chunks, mapping_chunk, mapping_message, mark_chunk,
+    iter_chunks, mapping_chunk, mapping_message, mark_chunk, owned_row_iter,
 };
 use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{
@@ -45,7 +44,7 @@ use crate::executor::{
 };
 use crate::task::{ActorId, CreateMviewProgress};
 
-/// An implementation of the RFC: Use Backfill To Let Mv On Mv Stream Again.(https://github.com/risingwavelabs/rfcs/pull/13)
+/// An implementation of the [RFC: Use Backfill To Let Mv On Mv Stream Again](https://github.com/risingwavelabs/rfcs/pull/13).
 /// `BackfillExecutor` is used to create a materialized view on another materialized view.
 ///
 /// It can only buffer chunks between two barriers instead of unbundled memory usage of
@@ -164,16 +163,20 @@ where
                 // It is finished, so just assign a value to avoid accessing storage table again.
                 false
             } else {
-                let snapshot = Self::snapshot_read(
-                    &self.upstream_table,
-                    init_epoch,
-                    None,
-                    false,
-                    self.chunk_size,
-                    &mut builder,
-                );
-                pin_mut!(snapshot);
-                snapshot.try_next().await?.unwrap().is_none()
+                let snapshot_is_empty = {
+                    let snapshot = Self::snapshot_read(
+                        &self.upstream_table,
+                        init_epoch,
+                        None,
+                        false,
+                        &mut builder,
+                    );
+                    pin_mut!(snapshot);
+                    snapshot.try_next().await?.unwrap().is_none()
+                };
+                let snapshot_buffer_is_empty = builder.is_empty();
+                builder.clear();
+                snapshot_is_empty && snapshot_buffer_is_empty
             }
         };
 
@@ -255,7 +258,6 @@ where
                         snapshot_read_epoch,
                         current_pos.clone(),
                         true,
-                        self.chunk_size,
                         &mut builder
                     )
                     .map(Either::Right),);
@@ -349,9 +351,9 @@ where
                 // - switch snapshot
 
                 // Consume snapshot rows left in builder
-                let chunk = builder.build_data_chunk();
-                let chunk_cardinality = chunk.cardinality() as u64;
-                if chunk_cardinality > 0 {
+                let chunk = builder.consume_all();
+                if let Some(chunk) = chunk {
+                    let chunk_cardinality = chunk.cardinality() as u64;
                     let ops = vec![Op::Insert; chunk.capacity()];
                     let chunk = StreamChunk::from_parts(ops, chunk);
                     current_pos = Some(get_new_pos(&chunk, &pk_in_output_indices));
@@ -477,13 +479,18 @@ where
         }
     }
 
+    /// Snapshot read the upstream mv.
+    /// The rows from upstream snapshot read will be buffered inside the `builder`.
+    /// If snapshot is dropped before its rows are consumed,
+    /// remaining data in `builder` must be flushed manually.
+    /// Otherwise when we scan a new snapshot, it is possible the rows in the `builder` would be
+    /// present, Then when we flush we contain duplicate rows.
     #[try_stream(ok = Option<StreamChunk>, error = StreamExecutorError)]
     async fn snapshot_read<'a>(
         upstream_table: &'a StorageTable<S>,
         epoch: u64,
         current_pos: Option<OwnedRow>,
         ordered: bool,
-        chunk_size: usize,
         builder: &'a mut DataChunkBuilder,
     ) {
         let range_bounds = compute_bounds(upstream_table.pk_indices(), current_pos);
@@ -505,13 +512,13 @@ where
                 ordered,
                 PrefetchOptions::new_for_exhaust_iter(),
             )
-            .await?
-            .map(get_second);
+            .await?;
 
-        pin_mut!(iter);
+        let row_iter = owned_row_iter(iter);
+        pin_mut!(row_iter);
 
         #[for_await]
-        for chunk in iter_chunks(iter, chunk_size, builder) {
+        for chunk in iter_chunks(row_iter, builder) {
             yield chunk?;
         }
     }
diff --git a/src/stream/src/executor/backfill/upstream_table/external.rs b/src/stream/src/executor/backfill/upstream_table/external.rs
new file mode 100644
index 0000000000000..08e22969a88a4
--- /dev/null
+++ b/src/stream/src/executor/backfill/upstream_table/external.rs
@@ -0,0 +1,106 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::catalog::{Schema, TableId};
+use risingwave_common::util::sort_util::OrderType;
+use risingwave_connector::source::external::{ExternalTableReaderImpl, SchemaTableName};
+
+/// This struct represents an external table to be read during backfill
+pub struct ExternalStorageTable {
+    /// Id for this table.
+    table_id: TableId,
+
+    /// The normalized name of the table, e.g. `dbname.schema_name.table_name`.
+    table_name: String,
+
+    schema_name: String,
+
+    table_reader: ExternalTableReaderImpl,
+
+    /// The schema of the output columns, i.e., this table VIEWED BY some executor like
+    /// RowSeqScanExecutor.
+    /// todo: the schema of the external table defined in the CREATE TABLE DDL
+    schema: Schema,
+
+    pk_order_types: Vec<OrderType>,
+
+    /// Indices of primary key.
+    /// Note that the index is based on the all columns of the table.
+    pk_indices: Vec<usize>,
+
+    output_indices: Vec<usize>,
+}
+
+impl ExternalStorageTable {
+    pub fn new(
+        table_id: TableId,
+        SchemaTableName {
+            table_name,
+            schema_name,
+        }: SchemaTableName,
+        table_reader: ExternalTableReaderImpl,
+        schema: Schema,
+        pk_order_types: Vec<OrderType>,
+        pk_indices: Vec<usize>,
+        output_indices: Vec<usize>,
+    ) -> Self {
+        Self {
+            table_id,
+            table_name,
+            schema_name,
+            table_reader,
+            schema,
+            pk_order_types,
+            pk_indices,
+            output_indices,
+        }
+    }
+
+    pub fn table_id(&self) -> TableId {
+        self.table_id
+    }
+
+    pub fn pk_order_types(&self) -> &[OrderType] {
+        &self.pk_order_types
+    }
+
+    pub fn schema(&self) -> &Schema {
+        &self.schema
+    }
+
+    pub fn pk_indices(&self) -> &[usize] {
+        &self.pk_indices
+    }
+
+    /// Get the indices of the primary key columns in the output columns.
+    ///
+    /// Returns `None` if any of the primary key columns is not in the output columns.
+    pub fn pk_in_output_indices(&self) -> Option<Vec<usize>> {
+        self.pk_indices
+            .iter()
+            .map(|&i| self.output_indices.iter().position(|&j| i == j))
+            .collect()
+    }
+
+    pub fn schema_table_name(&self) -> SchemaTableName {
+        SchemaTableName {
+            schema_name: self.schema_name.clone(),
+            table_name: self.table_name.clone(),
+        }
+    }
+
+    pub fn table_reader(&self) -> &ExternalTableReaderImpl {
+        &self.table_reader
+    }
+}
diff --git a/src/stream/src/executor/backfill/upstream_table/mod.rs b/src/stream/src/executor/backfill/upstream_table/mod.rs
new file mode 100644
index 0000000000000..8472def74266f
--- /dev/null
+++ b/src/stream/src/executor/backfill/upstream_table/mod.rs
@@ -0,0 +1,16 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod external;
+pub mod snapshot;
diff --git a/src/stream/src/executor/backfill/upstream_table/snapshot.rs b/src/stream/src/executor/backfill/upstream_table/snapshot.rs
new file mode 100644
index 0000000000000..806d78700154e
--- /dev/null
+++ b/src/stream/src/executor/backfill/upstream_table/snapshot.rs
@@ -0,0 +1,143 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+
+use futures::{pin_mut, Stream};
+use futures_async_stream::try_stream;
+use itertools::Itertools;
+use risingwave_common::array::StreamChunk;
+use risingwave_common::row::OwnedRow;
+use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+use risingwave_connector::source::external::{CdcOffset, ExternalTableReader};
+
+use crate::executor::backfill::upstream_table::external::ExternalStorageTable;
+use crate::executor::backfill::utils::iter_chunks;
+use crate::executor::{StreamExecutorResult, INVALID_EPOCH};
+
+pub trait UpstreamTableRead {
+    type BinlogOffsetFuture<'a>: Future<Output = StreamExecutorResult<Option<CdcOffset>>>
+        + Send
+        + 'a
+    where
+        Self: 'a;
+    type SnapshotStream<'a>: Stream<Item = StreamExecutorResult<Option<StreamChunk>>> + Send + 'a
+    where
+        Self: 'a;
+
+    fn snapshot_read(&self, args: SnapshotReadArgs) -> Self::SnapshotStream<'_>;
+
+    fn current_binlog_offset(&self) -> Self::BinlogOffsetFuture<'_>;
+}
+
+#[derive(Debug, Default)]
+pub struct SnapshotReadArgs {
+    pub epoch: u64,
+    pub current_pos: Option<OwnedRow>,
+    pub ordered: bool,
+    pub chunk_size: usize,
+}
+
+impl SnapshotReadArgs {
+    pub fn new(
+        epoch: u64,
+        current_pos: Option<OwnedRow>,
+        ordered: bool,
+        chunk_size: usize,
+    ) -> Self {
+        Self {
+            epoch,
+            current_pos,
+            ordered,
+            chunk_size,
+        }
+    }
+
+    pub fn new_for_cdc(current_pos: Option<OwnedRow>, chunk_size: usize) -> Self {
+        Self {
+            epoch: INVALID_EPOCH,
+            current_pos,
+            ordered: false,
+            chunk_size,
+        }
+    }
+}
+
+/// A wrapper of upstream table for snapshot read
+/// because we need to customize the snapshot read for managed upstream table (e.g. mv, index)
+/// and external upstream table.
+pub struct UpstreamTableReader<T> {
+    inner: T,
+}
+
+impl<T> UpstreamTableReader<T> {
+    pub fn inner(&self) -> &T {
+        &self.inner
+    }
+
+    pub fn new(table: T) -> Self {
+        Self { inner: table }
+    }
+}
+
+impl UpstreamTableRead for UpstreamTableReader<ExternalStorageTable> {
+    type BinlogOffsetFuture<'a> =
+        impl Future<Output = StreamExecutorResult<Option<CdcOffset>>> + 'a;
+    type SnapshotStream<'a> = impl Stream<Item = StreamExecutorResult<Option<StreamChunk>>> + 'a;
+
+    fn snapshot_read(&self, args: SnapshotReadArgs) -> Self::SnapshotStream<'_> {
+        #[try_stream]
+        async move {
+            let primary_keys = self
+                .inner
+                .pk_indices()
+                .iter()
+                .map(|idx| {
+                    let f = &self.inner.schema().fields[*idx];
+                    f.name.clone()
+                })
+                .collect_vec();
+
+            tracing::debug!(
+                "snapshot_read primary keys: {:?}, current_pos: {:?}",
+                primary_keys,
+                args.current_pos
+            );
+
+            let row_stream = self.inner.table_reader().snapshot_read(
+                self.inner.schema_table_name(),
+                args.current_pos,
+                primary_keys,
+            );
+
+            pin_mut!(row_stream);
+
+            let mut builder =
+                DataChunkBuilder::new(self.inner.schema().data_types(), args.chunk_size);
+            let chunk_stream = iter_chunks(row_stream, &mut builder);
+            #[for_await]
+            for chunk in chunk_stream {
+                yield chunk?;
+            }
+        }
+    }
+
+    fn current_binlog_offset(&self) -> Self::BinlogOffsetFuture<'_> {
+        async move {
+            let binlog = self.inner.table_reader().current_cdc_offset();
+            let binlog = binlog.await?;
+            Ok(Some(binlog))
+        }
+    }
+}
diff --git a/src/stream/src/executor/backfill/utils.rs b/src/stream/src/executor/backfill/utils.rs
index c19fe0f7f0fc9..6c49be9e607a1 100644
--- a/src/stream/src/executor/backfill/utils.rs
+++ b/src/stream/src/executor/backfill/utils.rs
@@ -17,8 +17,9 @@ use std::collections::HashMap;
 use std::ops::Bound;
 
 use await_tree::InstrumentAwait;
+use bytes::Bytes;
 use futures::future::try_join_all;
-use futures::Stream;
+use futures::{pin_mut, Stream, StreamExt};
 use futures_async_stream::try_stream;
 use risingwave_common::array::stream_record::Record;
 use risingwave_common::array::{Op, StreamChunk};
@@ -32,7 +33,11 @@ use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::iter_util::ZipEqDebug;
 use risingwave_common::util::sort_util::{cmp_datum_iter, OrderType};
 use risingwave_common::util::value_encoding::BasicSerde;
-use risingwave_storage::table::collect_data_chunk_with_builder;
+use risingwave_connector::error::ConnectorError;
+use risingwave_connector::source::external::{
+    CdcOffset, ExternalTableReader, ExternalTableReaderImpl,
+};
+use risingwave_storage::table::{collect_data_chunk_with_builder, KeyedRow};
 use risingwave_storage::StateStore;
 
 use crate::common::table::state_table::StateTableInner;
@@ -112,6 +117,25 @@ pub(crate) fn mark_chunk(
     mark_chunk_inner(chunk, current_pos, pk_in_output_indices, pk_order)
 }
 
+pub(crate) fn mark_cdc_chunk(
+    table_reader: &ExternalTableReaderImpl,
+    chunk: StreamChunk,
+    current_pos: &OwnedRow,
+    pk_in_output_indices: PkIndicesRef<'_>,
+    pk_order: &[OrderType],
+    last_cdc_offset: Option<CdcOffset>,
+) -> StreamExecutorResult<StreamChunk> {
+    let chunk = chunk.compact();
+    mark_cdc_chunk_inner(
+        table_reader,
+        chunk,
+        current_pos,
+        last_cdc_offset,
+        pk_in_output_indices,
+        pk_order,
+    )
+}
+
 /// Mark chunk:
 /// For each row of the chunk, forward it to downstream if its pk <= `current_pos` for the
 /// corresponding `vnode`, otherwise ignore it.
@@ -180,6 +204,55 @@ fn mark_chunk_inner(
     StreamChunk::new(ops, columns, Some(new_visibility.finish()))
 }
 
+fn mark_cdc_chunk_inner(
+    table_reader: &ExternalTableReaderImpl,
+    chunk: StreamChunk,
+    current_pos: &OwnedRow,
+    last_cdc_offset: Option<CdcOffset>,
+    pk_in_output_indices: PkIndicesRef<'_>,
+    pk_order: &[OrderType],
+) -> StreamExecutorResult<StreamChunk> {
+    let (data, ops) = chunk.into_parts();
+    let mut new_visibility = BitmapBuilder::with_capacity(ops.len());
+
+    // `_rw_offset` must be placed at the last column right now
+    let offset_col_idx = data.dimension() - 1;
+    for v in data.rows().map(|row| {
+        let offset_datum = row.datum_at(offset_col_idx).unwrap();
+        let event_offset = table_reader.parse_binlog_offset(offset_datum.into_utf8())?;
+        let visible = {
+            // filter changelog events with binlog range
+            let in_binlog_range = if let Some(binlog_low) = &last_cdc_offset {
+                binlog_low <= &event_offset
+            } else {
+                true
+            };
+
+            if in_binlog_range {
+                let lhs = row.project(pk_in_output_indices);
+                let rhs = current_pos;
+                let order = cmp_datum_iter(lhs.iter(), rhs.iter(), pk_order.iter().copied());
+                match order {
+                    Ordering::Less | Ordering::Equal => true,
+                    Ordering::Greater => false,
+                }
+            } else {
+                false
+            }
+        };
+        Ok::<_, ConnectorError>(visible)
+    }) {
+        new_visibility.append(v?);
+    }
+
+    let (columns, _) = data.into_parts();
+    Ok(StreamChunk::new(
+        ops,
+        columns,
+        Some(new_visibility.finish()),
+    ))
+}
+
 /// Builds a new stream chunk with `output_indices`.
 pub(crate) fn mapping_chunk(chunk: StreamChunk, output_indices: &[usize]) -> StreamChunk {
     let (ops, columns, visibility) = chunk.into_inner();
@@ -348,6 +421,18 @@ pub(crate) fn get_new_pos(chunk: &StreamChunk, pk_in_output_indices: &[usize]) -
         .into_owned_row()
 }
 
+pub(crate) fn get_cdc_chunk_last_offset(
+    table_reader: &ExternalTableReaderImpl,
+    chunk: &StreamChunk,
+) -> StreamExecutorResult<Option<CdcOffset>> {
+    let row = chunk.rows().last().unwrap().1;
+    let offset_col = row.iter().last().unwrap();
+    let output = offset_col.map(|scalar| {
+        Ok::<_, ConnectorError>(table_reader.parse_binlog_offset(scalar.into_utf8()))?
+    });
+    output.transpose().map_err(|e| e.into())
+}
+
 // NOTE(kwannoel): ["None" ..] encoding should be appropriate to mark
 // the case where upstream snapshot is empty.
 // This is so we can persist backfill state as "finished".
@@ -378,19 +463,28 @@ pub(crate) fn compute_bounds(
     }
 }
 
+#[try_stream(ok = OwnedRow, error = StreamExecutorError)]
+pub(crate) async fn owned_row_iter<S, E>(storage_iter: S)
+where
+    StreamExecutorError: From<E>,
+    S: Stream<Item = Result<KeyedRow<Bytes>, E>>,
+{
+    pin_mut!(storage_iter);
+    while let Some(row) = storage_iter.next().await {
+        let row = row?;
+        yield row.into_owned_row()
+    }
+}
+
 #[try_stream(ok = Option<StreamChunk>, error = StreamExecutorError)]
-pub(crate) async fn iter_chunks<'a, S, E>(
-    mut iter: S,
-    chunk_size: usize,
-    builder: &'a mut DataChunkBuilder,
-) where
+pub(crate) async fn iter_chunks<'a, S, E>(mut iter: S, builder: &'a mut DataChunkBuilder)
+where
     StreamExecutorError: From<E>,
     S: Stream<Item = Result<OwnedRow, E>> + Unpin + 'a,
 {
-    while let Some(data_chunk) =
-        collect_data_chunk_with_builder(&mut iter, Some(chunk_size), builder)
-            .instrument_await("backfill_snapshot_read")
-            .await?
+    while let Some(data_chunk) = collect_data_chunk_with_builder(&mut iter, builder)
+        .instrument_await("backfill_snapshot_read")
+        .await?
     {
         debug_assert!(data_chunk.cardinality() > 0);
         let ops = vec![Op::Insert; data_chunk.capacity()];
diff --git a/src/stream/src/executor/barrier_align.rs b/src/stream/src/executor/barrier_align.rs
index 423f0927e1e56..009c5e9b687c4 100644
--- a/src/stream/src/executor/barrier_align.rs
+++ b/src/stream/src/executor/barrier_align.rs
@@ -25,7 +25,7 @@ use risingwave_common::bail;
 use super::error::StreamExecutorError;
 use super::{Barrier, BoxedMessageStream, Message, StreamChunk, StreamExecutorResult, Watermark};
 use crate::executor::monitor::StreamingMetrics;
-use crate::task::ActorId;
+use crate::task::{ActorId, FragmentId};
 
 pub type AlignedMessageStreamItem = StreamExecutorResult<AlignedMessage>;
 pub trait AlignedMessageStream = futures::Stream<Item = AlignedMessageStreamItem> + Send;
@@ -44,9 +44,11 @@ pub async fn barrier_align(
     mut left: BoxedMessageStream,
     mut right: BoxedMessageStream,
     actor_id: ActorId,
+    fragment_id: FragmentId,
     metrics: Arc<StreamingMetrics>,
 ) {
     let actor_id = actor_id.to_string();
+    let fragment_id = fragment_id.to_string();
     loop {
         let prefer_left: bool = rand::random();
         let select_result = if prefer_left {
@@ -107,7 +109,7 @@ pub async fn barrier_align(
                             yield AlignedMessage::Barrier(barrier);
                             metrics
                                 .join_barrier_align_duration
-                                .with_label_values(&[&actor_id, "right"])
+                                .with_label_values(&[&actor_id, &fragment_id, "right"])
                                 .observe(start_time.elapsed().as_secs_f64());
                             break;
                         }
@@ -133,7 +135,7 @@ pub async fn barrier_align(
                             yield AlignedMessage::Barrier(barrier);
                             metrics
                                 .join_barrier_align_duration
-                                .with_label_values(&[&actor_id, "left"])
+                                .with_label_values(&[&actor_id, &fragment_id, "left"])
                                 .observe(start_time.elapsed().as_secs_f64());
                             break;
                         }
@@ -159,7 +161,7 @@ mod tests {
         left: BoxedMessageStream,
         right: BoxedMessageStream,
     ) -> impl Stream<Item = Result<AlignedMessage, StreamExecutorError>> {
-        barrier_align(left, right, 0, Arc::new(StreamingMetrics::unused()))
+        barrier_align(left, right, 0, 0, Arc::new(StreamingMetrics::unused()))
     }
 
     #[tokio::test]
diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs
index 5929702d2dd1d..12f6ff736dc5e 100644
--- a/src/stream/src/executor/batch_query.rs
+++ b/src/stream/src/executor/batch_query.rs
@@ -20,7 +20,7 @@ use risingwave_common::catalog::Schema;
 use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::table::batch_table::storage_table::StorageTable;
-use risingwave_storage::table::{collect_data_chunk, get_second};
+use risingwave_storage::table::collect_data_chunk;
 use risingwave_storage::StateStore;
 
 use super::error::StreamExecutorError;
@@ -58,8 +58,7 @@ where
                 false,
                 PrefetchOptions::new_for_exhaust_iter(),
             )
-            .await?
-            .map(get_second);
+            .await?;
         pin_mut!(iter);
 
         while let Some(data_chunk) =
diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs
index 56a10da734d37..ab3ef9ae44973 100644
--- a/src/stream/src/executor/chain.rs
+++ b/src/stream/src/executor/chain.rs
@@ -178,6 +178,7 @@ mod test {
                     },
                     added_actors: maplit::hashset! { actor_id },
                     splits: Default::default(),
+                    pause: false,
                 })),
                 Message::Chunk(StreamChunk::from_pretty("I\n + 3")),
                 Message::Chunk(StreamChunk::from_pretty("I\n + 4")),
diff --git a/src/stream/src/executor/dispatch.rs b/src/stream/src/executor/dispatch.rs
index ec2bfec01c032..17b8866543c6f 100644
--- a/src/stream/src/executor/dispatch.rs
+++ b/src/stream/src/executor/dispatch.rs
@@ -624,7 +624,7 @@ impl Dispatcher for HashDataDispatcher {
             // get hash value of every line by its key
             let vnodes = VirtualNode::compute_chunk(chunk.data_chunk(), &self.keys);
 
-            tracing::trace!(target: "events::stream::dispatch::hash", "\n{}\n keys {:?} => {:?}", chunk.to_pretty_string(), self.keys, vnodes);
+            tracing::trace!(target: "events::stream::dispatch::hash", "\n{}\n keys {:?} => {:?}", chunk.to_pretty(), self.keys, vnodes);
 
             let mut vis_maps = repeat_with(|| BitmapBuilder::with_capacity(chunk.capacity()))
                 .take(num_outputs)
diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs
index 16ae74841b7f0..b324f4666cc2e 100644
--- a/src/stream/src/executor/dml.rs
+++ b/src/stream/src/executor/dml.rs
@@ -119,9 +119,8 @@ impl DmlExecutor {
         // a round robin way.
         let mut stream = StreamReaderWithPause::<false, TxnMsg>::new(upstream, batch_reader);
 
-        // If the first barrier is configuration change, then the DML executor must be newly
-        // created, and we should start with the paused state.
-        if barrier.is_update() {
+        // If the first barrier requires us to pause on startup, pause the stream.
+        if barrier.is_pause_on_startup() {
             stream.pause_stream();
         }
 
diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs
index 4a9ce3bd565f8..234ef6db29ab2 100644
--- a/src/stream/src/executor/dynamic_filter.rs
+++ b/src/stream/src/executor/dynamic_filter.rs
@@ -232,11 +232,11 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
 
     async fn recover_rhs(&mut self) -> Result<Option<RowData>, StreamExecutorError> {
         // Recover value for RHS if available
-        let rhs_stream = self.right_table.iter(Default::default()).await?;
+        let rhs_stream = self.right_table.iter_row(Default::default()).await?;
         pin_mut!(rhs_stream);
 
         if let Some(res) = rhs_stream.next().await {
-            let value = res?;
+            let value = res?.into_owned_row();
             assert!(rhs_stream.next().await.is_none());
             Ok(Some(value))
         } else {
@@ -253,7 +253,6 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         let input_l = self.source_l.take().unwrap();
         let input_r = self.source_r.take().unwrap();
 
-        let left_len = input_l.schema().len();
         // Derive the dynamic expression
         let l_data_type = input_l.schema().data_types()[self.key_l].clone();
         let r_data_type = input_r.schema().data_types()[0].clone();
@@ -276,6 +275,7 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
             input_l.execute(),
             input_r.execute(),
             self.ctx.id,
+            self.ctx.fragment_id,
             self.metrics.clone(),
         );
 
@@ -297,14 +297,8 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         // The first barrier message should be propagated.
         yield Message::Barrier(barrier);
 
-        let (left_to_output, _) =
-            StreamChunkBuilder::get_i2o_mapping(0..self.schema.len(), left_len, 0);
-        let mut stream_chunk_builder = StreamChunkBuilder::new(
-            self.chunk_size,
-            &self.schema.data_types(),
-            vec![],
-            left_to_output,
-        );
+        let mut stream_chunk_builder =
+            StreamChunkBuilder::new(self.chunk_size, self.schema.data_types());
 
         let watermark_can_clean_state = !matches!(self.comparator, LessThan | LessThanOrEqual);
         let mut unused_clean_hint = None;
@@ -387,7 +381,7 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                         // TODO: prefetching for append-only case.
                         let streams = futures::future::try_join_all(
                             self.left_table.vnodes().iter_vnodes().map(|vnode| {
-                                self.left_table.iter_with_pk_range(
+                                self.left_table.iter_row_with_pk_range(
                                     &range,
                                     vnode,
                                     PrefetchOptions::new_for_exhaust_iter(),
@@ -401,10 +395,10 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                         #[for_await]
                         for res in stream::select_all(streams) {
                             let row = res?;
-                            if let Some(chunk) = stream_chunk_builder.append_row_matched(
+                            if let Some(chunk) = stream_chunk_builder.append_row(
                                 // All rows have a single identity at this point
                                 if is_insert { Op::Insert } else { Op::Delete },
-                                row,
+                                row.as_ref(),
                             ) {
                                 yield Message::Chunk(chunk);
                             }
diff --git a/src/stream/src/executor/error.rs b/src/stream/src/executor/error.rs
index c4112cc03bd58..59be30791b966 100644
--- a/src/stream/src/executor/error.rs
+++ b/src/stream/src/executor/error.rs
@@ -220,6 +220,12 @@ impl From<PbFieldNotFound> for StreamExecutorError {
     }
 }
 
+impl From<String> for StreamExecutorError {
+    fn from(s: String) -> Self {
+        ErrorKind::Internal(anyhow::anyhow!(s)).into()
+    }
+}
+
 static_assertions::const_assert_eq!(std::mem::size_of::<StreamExecutorError>(), 8);
 
 #[cfg(test)]
diff --git a/src/stream/src/executor/exchange/permit.rs b/src/stream/src/executor/exchange/permit.rs
index 11cbbe2522323..9c333ddf4b361 100644
--- a/src/stream/src/executor/exchange/permit.rs
+++ b/src/stream/src/executor/exchange/permit.rs
@@ -113,7 +113,7 @@ pub struct Sender {
     permits: Arc<Permits>,
 
     /// The maximum permits required by a chunk. If there're too many rows in a chunk, we only
-    /// acquire these permits. [`BATCHED_PERMITS`] is subtracted to avoid deadlock with
+    /// acquire these permits. `BATCHED_PERMITS` is subtracted to avoid deadlock with
     /// batching.
     max_chunk_permits: usize,
 }
diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs
new file mode 100644
index 0000000000000..1790f212566b7
--- /dev/null
+++ b/src/stream/src/executor/flow_control.rs
@@ -0,0 +1,98 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{Debug, Formatter};
+use std::num::NonZeroU32;
+
+use governor::clock::MonotonicClock;
+use governor::{Quota, RateLimiter};
+use risingwave_common::catalog::Schema;
+
+use super::*;
+
+/// Flow Control Executor is used to control the rate of the input executor.
+///
+/// Currently it is placed after the `BackfillExecutor`:
+/// upstream `MaterializeExecutor` -> `BackfillExecutor` -> `FlowControlExecutor`
+///
+/// The rate limit is set statically at the moment, and cannot be changed in a running
+/// stream graph.
+///
+/// It is used to throttle problematic MVs that are consuming too much resources.
+pub struct FlowControlExecutor {
+    input: BoxedExecutor,
+    rate_limit: u32,
+}
+
+impl FlowControlExecutor {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(input: Box<dyn Executor>, rate_limit: u32) -> Self {
+        #[cfg(madsim)]
+        println!("FlowControlExecutor rate limiter is disabled in madsim as it will spawn system threads");
+        Self { input, rate_limit }
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn execute_inner(self) {
+        let quota = Quota::per_second(NonZeroU32::new(self.rate_limit).unwrap());
+        let clock = MonotonicClock;
+        let rate_limiter = RateLimiter::direct_with_clock(quota, &clock);
+        #[for_await]
+        for msg in self.input.execute() {
+            let msg = msg?;
+            match msg {
+                Message::Chunk(chunk) => {
+                    #[cfg(not(madsim))]
+                    {
+                        let result = rate_limiter
+                            .until_n_ready(NonZeroU32::new(chunk.cardinality() as u32).unwrap())
+                            .await;
+                        assert!(
+                            result.is_ok(),
+                            "the capacity of rate_limiter must be larger than the cardinality of chunk"
+                        );
+                    }
+                    yield Message::Chunk(chunk);
+                }
+                _ => yield msg,
+            }
+        }
+    }
+}
+
+impl Debug for FlowControlExecutor {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("FlowControlExecutor")
+            .field("rate_limit", &self.rate_limit)
+            .finish()
+    }
+}
+
+impl Executor for FlowControlExecutor {
+    fn execute(self: Box<Self>) -> BoxedMessageStream {
+        self.execute_inner().boxed()
+    }
+
+    fn schema(&self) -> &Schema {
+        self.input.schema()
+    }
+
+    fn pk_indices(&self) -> PkIndicesRef<'_> {
+        self.input.pk_indices()
+    }
+
+    fn identity(&self) -> &str {
+        "FlowControlExecutor"
+    }
+}
diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs
index 56130becf322c..610c2857728b3 100644
--- a/src/stream/src/executor/hash_agg.rs
+++ b/src/stream/src/executor/hash_agg.rs
@@ -32,8 +32,8 @@ use risingwave_storage::StateStore;
 
 use super::agg_common::{AggExecutorArgs, HashAggExecutorExtraArgs};
 use super::aggregation::{
-    agg_call_filter_res, iter_table_storage, AggStateStorage, ChunkBuilder, DistinctDeduplicater,
-    GroupKey, OnlyOutputIfHasInput,
+    agg_call_filter_res, iter_table_storage, AggStateStorage, DistinctDeduplicater, GroupKey,
+    OnlyOutputIfHasInput,
 };
 use super::sort_buffer::SortBuffer;
 use super::{
@@ -43,6 +43,7 @@ use super::{
 use crate::cache::{cache_may_stale, new_with_hasher, ManagedLruCache};
 use crate::common::metrics::MetricsInfo;
 use crate::common::table::state_table::StateTable;
+use crate::common::StreamChunkBuilder;
 use crate::error::StreamResult;
 use crate::executor::aggregation::{generate_agg_schema, AggGroup as GenericAggGroup};
 use crate::executor::error::StreamExecutorError;
@@ -152,7 +153,7 @@ struct ExecutionVars<K: HashKey, S: StateStore> {
     window_watermark: Option<ScalarImpl>,
 
     /// Stream chunk builder.
-    chunk_builder: ChunkBuilder,
+    chunk_builder: StreamChunkBuilder,
 
     buffer: SortBuffer<S>,
 }
@@ -563,7 +564,7 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
             ),
             buffered_watermarks: vec![None; this.group_key_indices.len()],
             window_watermark: None,
-            chunk_builder: ChunkBuilder::new(this.chunk_size, &this.info.schema.data_types()),
+            chunk_builder: StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types()),
             buffer: SortBuffer::new(window_col_idx_in_group_key, &this.result_table),
         };
 
diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs
index 00832b3e13858..31705e792909a 100644
--- a/src/stream/src/executor/hash_join.rs
+++ b/src/stream/src/executor/hash_join.rs
@@ -44,7 +44,7 @@ use super::{
     Watermark,
 };
 use crate::common::table::state_table::StateTable;
-use crate::common::StreamChunkBuilder;
+use crate::common::JoinStreamChunkBuilder;
 use crate::executor::expect_first_barrier_from_aligned_stream;
 use crate::executor::JoinType::LeftAnti;
 use crate::task::AtomicU64Ref;
@@ -305,7 +305,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> Executor for HashJoi
 }
 
 struct HashJoinChunkBuilder<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> {
-    stream_chunk_builder: StreamChunkBuilder,
+    stream_chunk_builder: JoinStreamChunkBuilder,
 }
 
 struct EqJoinArgs<'a, K: HashKey, S: StateStore> {
@@ -475,7 +475,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
 
         let original_output_data_types = schema_fields
             .iter()
-            .map(|field| field.data_type.clone())
+            .map(|field| field.data_type())
             .collect_vec();
         let actual_output_data_types = output_indices
             .iter()
@@ -483,18 +483,8 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             .collect_vec();
 
         // Data types of of hash join state.
-        let state_all_data_types_l = input_l
-            .schema()
-            .fields
-            .iter()
-            .map(|field| field.data_type.clone())
-            .collect_vec();
-        let state_all_data_types_r = input_r
-            .schema()
-            .fields
-            .iter()
-            .map(|field| field.data_type.clone())
-            .collect_vec();
+        let state_all_data_types_l = input_l.schema().data_types();
+        let state_all_data_types_r = input_r.schema().data_types();
 
         let state_pk_indices_l = input_l.pk_indices().to_vec();
         let state_pk_indices_r = input_r.pk_indices().to_vec();
@@ -519,12 +509,12 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
         // check whether join key contains pk in both side
         let append_only_optimize = is_append_only && pk_contained_in_jk_l && pk_contained_in_jk_r;
 
-        let join_key_data_types_r = join_key_indices_l
+        let join_key_data_types_l = join_key_indices_l
             .iter()
             .map(|idx| state_all_data_types_l[*idx].clone())
             .collect_vec();
 
-        let join_key_data_types_l = join_key_indices_r
+        let join_key_data_types_r = join_key_indices_r
             .iter()
             .map(|idx| state_all_data_types_r[*idx].clone())
             .collect_vec();
@@ -561,7 +551,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             } else {
                 (state_all_data_types_l.len(), state_all_data_types_r.len())
             };
-            StreamChunkBuilder::get_i2o_mapping(output_indices.iter().cloned(), left_len, right_len)
+            JoinStreamChunkBuilder::get_i2o_mapping(&output_indices, left_len, right_len)
         };
 
         let l2o_indexed = MultiMap::from_iter(left_to_output.iter().copied());
@@ -711,6 +701,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             input_l.execute(),
             input_r.execute(),
             self.ctx.id,
+            self.ctx.fragment_id,
             self.metrics.clone(),
         );
         pin_mut!(aligned_stream);
@@ -1030,9 +1021,9 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             .collect_vec();
 
         let mut hashjoin_chunk_builder = HashJoinChunkBuilder::<T, SIDE> {
-            stream_chunk_builder: StreamChunkBuilder::new(
+            stream_chunk_builder: JoinStreamChunkBuilder::new(
                 chunk_size,
-                actual_output_data_types,
+                actual_output_data_types.to_vec(),
                 side_update.i2o_mapping.clone(),
                 side_match.i2o_mapping.clone(),
             ),
@@ -1041,7 +1032,11 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
         let join_matched_join_keys = ctx
             .streaming_metrics
             .join_matched_join_keys
-            .with_label_values(&[&ctx.id.to_string(), &side_update.ht.table_id().to_string()]);
+            .with_label_values(&[
+                &ctx.id.to_string(),
+                &ctx.fragment_id.to_string(),
+                &side_update.ht.table_id().to_string(),
+            ]);
 
         let keys = K::build(&side_update.join_key_indices, chunk.data_chunk())?;
         for ((op, row), key) in chunk.rows().zip_eq_debug(keys.iter()) {
diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs
index 8d1450cd883d3..aa1840aa832ce 100644
--- a/src/stream/src/executor/hop_window.rs
+++ b/src/stream/src/executor/hop_window.rs
@@ -17,14 +17,14 @@ use std::num::NonZeroUsize;
 use futures::StreamExt;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
-use risingwave_common::array::{DataChunk, Op, StreamChunk, Vis};
+use risingwave_common::array::{DataChunk, Op, Vis};
 use risingwave_common::types::Interval;
-use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_expr::expr::BoxedExpression;
 use risingwave_expr::ExprError;
 
 use super::error::StreamExecutorError;
 use super::{ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message};
+use crate::common::StreamChunkBuilder;
 
 pub struct HopWindowExecutor {
     ctx: ActorContextRef,
@@ -193,8 +193,7 @@ impl HopWindowExecutor {
                     let mut row_iters = chunks.iter().map(|c| c.rows()).collect_vec();
 
                     let data_types = chunks[0].data_types();
-                    let mut chunk_builder = DataChunkBuilder::new(data_types, chunk_size);
-                    let mut op_builder = Vec::with_capacity(chunk_size);
+                    let mut chunk_builder = StreamChunkBuilder::new(chunk_size, data_types);
 
                     for &op in &*ops {
                         // Since there could be multiple rows for the same input row, we need to
@@ -204,25 +203,19 @@ impl HopWindowExecutor {
                             Op::Delete | Op::UpdateDelete => Op::Delete,
                         };
                         for row_iter in &mut row_iters {
-                            op_builder.push(op);
                             if let Some(chunk) =
-                                chunk_builder.append_one_row(row_iter.next().unwrap())
+                                chunk_builder.append_row(op, row_iter.next().unwrap())
                             {
-                                let ops = op_builder.drain(..).collect_vec();
-                                let chunk = StreamChunk::from_parts(ops, chunk);
                                 yield Message::Chunk(chunk);
                             }
                         }
                     }
 
-                    if let Some(chunk) = chunk_builder.consume_all() {
-                        let ops = op_builder.drain(..).collect_vec();
-                        let chunk = StreamChunk::from_parts(ops, chunk);
+                    if let Some(chunk) = chunk_builder.take() {
                         yield Message::Chunk(chunk);
                     }
 
-                    // All builders should be exhausted.
-                    debug_assert!(op_builder.is_empty());
+                    // Rows should be exhausted.
                     debug_assert!(row_iters.into_iter().all(|mut it| it.next().is_none()));
                 }
                 Message::Barrier(b) => {
diff --git a/src/stream/src/executor/lookup/impl_.rs b/src/stream/src/executor/lookup/impl_.rs
index 866aca20e9753..463cec2d5f6bf 100644
--- a/src/stream/src/executor/lookup/impl_.rs
+++ b/src/stream/src/executor/lookup/impl_.rs
@@ -31,7 +31,7 @@ use risingwave_storage::StateStore;
 use super::sides::{stream_lookup_arrange_prev_epoch, stream_lookup_arrange_this_epoch};
 use crate::cache::cache_may_stale;
 use crate::common::metrics::MetricsInfo;
-use crate::common::StreamChunkBuilder;
+use crate::common::JoinStreamChunkBuilder;
 use crate::executor::error::{StreamExecutorError, StreamExecutorResult};
 use crate::executor::lookup::cache::LookupCache;
 use crate::executor::lookup::sides::{ArrangeJoinSide, ArrangeMessage, StreamJoinSide};
@@ -264,8 +264,8 @@ impl<S: StateStore> LookupExecutor<S> {
             .boxed()
         };
 
-        let (stream_to_output, arrange_to_output) = StreamChunkBuilder::get_i2o_mapping(
-            self.column_mapping.iter().cloned(),
+        let (stream_to_output, arrange_to_output) = JoinStreamChunkBuilder::get_i2o_mapping(
+            &self.column_mapping,
             self.stream.col_types.len(),
             self.arrangement.col_types.len(),
         );
@@ -310,9 +310,9 @@ impl<S: StateStore> LookupExecutor<S> {
                     let chunk = chunk.compact();
                     let (chunk, ops) = chunk.into_parts();
 
-                    let mut builder = StreamChunkBuilder::new(
+                    let mut builder = JoinStreamChunkBuilder::new(
                         self.chunk_size,
-                        &reorder_chunk_data_types,
+                        reorder_chunk_data_types.clone(),
                         stream_to_output.clone(),
                         arrange_to_output.clone(),
                     );
diff --git a/src/stream/src/executor/managed_state/join/mod.rs b/src/stream/src/executor/managed_state/join/mod.rs
index 1e9409d596d09..f811d661d364a 100644
--- a/src/stream/src/executor/managed_state/join/mod.rs
+++ b/src/stream/src/executor/managed_state/join/mod.rs
@@ -405,19 +405,21 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
             let table_iter_fut = self
                 .state
                 .table
-                .iter_key_and_val(&key, PrefetchOptions::new_for_exhaust_iter());
+                .iter_row_with_pk_prefix(&key, PrefetchOptions::new_for_exhaust_iter());
             let degree_table_iter_fut = self
                 .degree_state
                 .table
-                .iter_key_and_val(&key, PrefetchOptions::new_for_exhaust_iter());
+                .iter_row_with_pk_prefix(&key, PrefetchOptions::new_for_exhaust_iter());
 
             let (table_iter, degree_table_iter) =
                 try_join(table_iter_fut, degree_table_iter_fut).await?;
 
             #[for_await]
             for (row, degree) in table_iter.zip(degree_table_iter) {
-                let (pk1, row) = row?;
-                let (pk2, degree) = degree?;
+                let row = row?;
+                let degree_row = degree?;
+                let pk1 = row.key();
+                let pk2 = degree_row.key();
                 debug_assert_eq!(
                     pk1, pk2,
                     "mismatched pk in degree table: pk1: {pk1:?}, pk2: {pk2:?}",
@@ -426,29 +428,29 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
                     .as_ref()
                     .project(&self.state.pk_indices)
                     .memcmp_serialize(&self.pk_serializer);
-                let degree_i64 = degree
-                    .datum_at(degree.len() - 1)
+                let degree_i64 = degree_row
+                    .datum_at(degree_row.len() - 1)
                     .expect("degree should not be NULL");
                 entry_state.insert(
                     pk,
-                    JoinRow::new(row, degree_i64.into_int64() as u64).encode(),
+                    JoinRow::new(row.into_owned_row(), degree_i64.into_int64() as u64).encode(),
                 );
             }
         } else {
             let table_iter = self
                 .state
                 .table
-                .iter_with_pk_prefix(&key, PrefetchOptions::new_for_exhaust_iter())
+                .iter_row_with_pk_prefix(&key, PrefetchOptions::new_for_exhaust_iter())
                 .await?;
 
             #[for_await]
-            for row in table_iter {
-                let row: OwnedRow = row?;
+            for entry in table_iter {
+                let row = entry?;
                 let pk = row
                     .as_ref()
                     .project(&self.state.pk_indices)
                     .memcmp_serialize(&self.pk_serializer);
-                entry_state.insert(pk, JoinRow::new(row, 0).encode());
+                entry_state.insert(pk, JoinRow::new(row.into_owned_row(), 0).encode());
             }
         };
 
diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs
index 49a7f4001decd..dad7bf4ea3abb 100644
--- a/src/stream/src/executor/merge.rs
+++ b/src/stream/src/executor/merge.rs
@@ -138,7 +138,7 @@ impl MergeExecutor {
                 }
                 Message::Barrier(barrier) => {
                     tracing::trace!(
-                        target: "events::barrier::path",
+                        target: "events::stream::barrier::path",
                         actor_id = actor_id,
                         "receiver receives barrier from path: {:?}",
                         barrier.passed_actors
diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs
index b04865c420ab2..8fa7a5d818cc4 100644
--- a/src/stream/src/executor/mod.rs
+++ b/src/stream/src/executor/mod.rs
@@ -65,6 +65,7 @@ mod dynamic_filter;
 mod error;
 mod expand;
 mod filter;
+mod flow_control;
 mod hash_agg;
 pub mod hash_join;
 mod hop_window;
@@ -103,7 +104,9 @@ pub mod test_utils;
 
 pub use actor::{Actor, ActorContext, ActorContextRef};
 use anyhow::Context;
+pub use backfill::cdc_backfill::*;
 pub use backfill::no_shuffle_backfill::*;
+pub use backfill::upstream_table::*;
 pub use barrier_recv::BarrierRecvExecutor;
 pub use batch_query::BatchQueryExecutor;
 pub use chain::ChainExecutor;
@@ -113,6 +116,7 @@ pub use dynamic_filter::DynamicFilterExecutor;
 pub use error::{StreamExecutorError, StreamExecutorResult};
 pub use expand::ExpandExecutor;
 pub use filter::FilterExecutor;
+pub use flow_control::FlowControlExecutor;
 pub use hash_agg::HashAggExecutor;
 pub use hash_join::*;
 pub use hop_window::HopWindowExecutor;
@@ -231,6 +235,7 @@ pub enum Mutation {
         added_actors: HashSet<ActorId>,
         // TODO: remove this and use `SourceChangesSplit` after we support multiple mutations.
         splits: HashMap<ActorId, Vec<SplitImpl>>,
+        pause: bool,
     },
     SourceChangeSplit(HashMap<ActorId, Vec<SplitImpl>>),
     Pause,
@@ -317,9 +322,15 @@ impl Barrier {
         }
     }
 
-    /// Whether this barrier is for pause.
-    pub fn is_pause(&self) -> bool {
-        matches!(self.mutation.as_deref(), Some(Mutation::Pause))
+    /// Whether this barrier requires the executor to pause its data stream on startup.
+    pub fn is_pause_on_startup(&self) -> bool {
+        match self.mutation.as_deref() {
+            Some(
+                  Mutation::Update { .. } // new actors for scaling
+                | Mutation::Add { pause: true, .. } // new streaming job, or recovery
+            ) => true,
+            _ => false,
+        }
     }
 
     /// Whether this barrier is for configuration change. Used for source executor initialization.
@@ -438,6 +449,7 @@ impl Mutation {
                 adds,
                 added_actors,
                 splits,
+                pause,
             } => PbMutation::Add(AddMutation {
                 actor_dispatchers: adds
                     .iter()
@@ -452,6 +464,7 @@ impl Mutation {
                     .collect(),
                 added_actors: added_actors.iter().copied().collect(),
                 actor_splits: actor_splits_to_protobuf(splits),
+                pause: *pause,
             }),
             Mutation::SourceChangeSplit(changes) => PbMutation::Splits(SourceChangeSplitMutation {
                 actor_splits: changes
@@ -536,6 +549,7 @@ impl Mutation {
                         )
                     })
                     .collect(),
+                pause: add.pause,
             },
 
             PbMutation::Splits(s) => {
diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs
index dde0437bf76b3..2fc0c1b8b3b7d 100644
--- a/src/stream/src/executor/monitor/streaming_stats.rs
+++ b/src/stream/src/executor/monitor/streaming_stats.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::LazyLock;
+use std::sync::OnceLock;
 
 use prometheus::core::{AtomicF64, AtomicI64, AtomicU64, GenericCounterVec, GenericGaugeVec};
 use prometheus::{
@@ -22,10 +22,14 @@ use prometheus::{
     register_int_gauge_vec_with_registry, register_int_gauge_with_registry, Histogram,
     HistogramVec, IntCounter, IntGauge, Registry,
 };
+use risingwave_common::config::MetricLevel;
+use risingwave_common::metrics::RelabeledHistogramVec;
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 
 #[derive(Clone)]
 pub struct StreamingMetrics {
+    pub level: MetricLevel,
+
     pub executor_row_count: GenericCounterVec<AtomicU64>,
     pub actor_execution_time: GenericGaugeVec<AtomicF64>,
     pub actor_output_buffer_blocking_duration_ns: GenericCounterVec<AtomicU64>,
@@ -57,11 +61,11 @@ pub struct StreamingMetrics {
     pub join_insert_cache_miss_count: GenericCounterVec<AtomicU64>,
     pub join_actor_input_waiting_duration_ns: GenericCounterVec<AtomicU64>,
     pub join_match_duration_ns: GenericCounterVec<AtomicU64>,
-    pub join_barrier_align_duration: HistogramVec,
+    pub join_barrier_align_duration: RelabeledHistogramVec,
     pub join_cached_entries: GenericGaugeVec<AtomicI64>,
     pub join_cached_rows: GenericGaugeVec<AtomicI64>,
     pub join_cached_estimated_size: GenericGaugeVec<AtomicI64>,
-    pub join_matched_join_keys: HistogramVec,
+    pub join_matched_join_keys: RelabeledHistogramVec,
 
     // Streaming Aggregation
     pub agg_lookup_miss_count: GenericCounterVec<AtomicU64>,
@@ -99,12 +103,19 @@ pub struct StreamingMetrics {
     pub arrangement_backfill_snapshot_read_row_count: GenericCounterVec<AtomicU64>,
     pub arrangement_backfill_upstream_output_row_count: GenericCounterVec<AtomicU64>,
 
+    // Over Window
+    pub over_window_cached_entry_count: GenericGaugeVec<AtomicI64>,
+    pub over_window_cache_lookup_count: GenericCounterVec<AtomicU64>,
+    pub over_window_cache_miss_count: GenericCounterVec<AtomicU64>,
+
     /// The duration from receipt of barrier to all actors collection.
     /// And the max of all node `barrier_inflight_latency` is the latency for a barrier
     /// to flow through the graph.
     pub barrier_inflight_latency: Histogram,
     /// The duration of sync to storage.
     pub barrier_sync_latency: Histogram,
+    /// The progress made by the earliest in-flight barriers in the local barrier manager.
+    pub barrier_manager_progress: IntCounter,
 
     pub sink_commit_duration: HistogramVec,
 
@@ -121,6 +132,9 @@ pub struct StreamingMetrics {
     /// User compute error reporting
     pub user_compute_error_count: GenericCounterVec<AtomicU64>,
 
+    /// User source reader error
+    pub user_source_reader_error_count: GenericCounterVec<AtomicU64>,
+
     // Materialize
     pub materialize_cache_hit_count: GenericCounterVec<AtomicU64>,
     pub materialize_cache_total_count: GenericCounterVec<AtomicU64>,
@@ -129,11 +143,16 @@ pub struct StreamingMetrics {
     pub stream_memory_usage: GenericGaugeVec<AtomicI64>,
 }
 
-pub static GLOBAL_STREAMING_METRICS: LazyLock<StreamingMetrics> =
-    LazyLock::new(|| StreamingMetrics::new(&GLOBAL_METRICS_REGISTRY));
+pub static GLOBAL_STREAMING_METRICS: OnceLock<StreamingMetrics> = OnceLock::new();
+
+pub fn global_streaming_metrics(metric_level: MetricLevel) -> StreamingMetrics {
+    GLOBAL_STREAMING_METRICS
+        .get_or_init(|| StreamingMetrics::new(&GLOBAL_METRICS_REGISTRY, metric_level))
+        .clone()
+}
 
 impl StreamingMetrics {
-    fn new(registry: &Registry) -> Self {
+    fn new(registry: &Registry, level: MetricLevel) -> Self {
         let executor_row_count = register_int_counter_vec_with_registry!(
             "stream_executor_row_count",
             "Total number of rows that have been output from each executor",
@@ -355,9 +374,19 @@ impl StreamingMetrics {
             "Duration of join align barrier",
             exponential_buckets(0.0001, 2.0, 21).unwrap() // max 104s
         );
-        let join_barrier_align_duration =
-            register_histogram_vec_with_registry!(opts, &["actor_id", "wait_side"], registry)
-                .unwrap();
+        let join_barrier_align_duration = register_histogram_vec_with_registry!(
+            opts,
+            &["actor_id", "fragment_id", "wait_side"],
+            registry
+        )
+        .unwrap();
+
+        let join_barrier_align_duration = RelabeledHistogramVec::with_metric_level_relabel_n(
+            MetricLevel::Debug,
+            join_barrier_align_duration,
+            level,
+            1,
+        );
 
         let join_cached_entries = register_int_gauge_vec_with_registry!(
             "stream_join_cached_entries",
@@ -391,11 +420,18 @@ impl StreamingMetrics {
 
         let join_matched_join_keys = register_histogram_vec_with_registry!(
             join_matched_join_keys_opts,
-            &["actor_id", "table_id"],
+            &["actor_id", "fragment_id", "table_id"],
             registry
         )
         .unwrap();
 
+        let join_matched_join_keys = RelabeledHistogramVec::with_metric_level_relabel_n(
+            MetricLevel::Debug,
+            join_matched_join_keys,
+            level,
+            1,
+        );
+
         let agg_lookup_miss_count = register_int_counter_vec_with_registry!(
             "stream_agg_lookup_miss_count",
             "Aggregation executor lookup miss duration",
@@ -590,6 +626,30 @@ impl StreamingMetrics {
             )
             .unwrap();
 
+        let over_window_cached_entry_count = register_int_gauge_vec_with_registry!(
+            "stream_over_window_cached_entry_count",
+            "Total entry (partition) count in over window executor cache",
+            &["table_id", "actor_id"],
+            registry
+        )
+        .unwrap();
+
+        let over_window_cache_lookup_count = register_int_counter_vec_with_registry!(
+            "stream_over_window_cache_lookup_count",
+            "Over window executor cache lookup count",
+            &["table_id", "actor_id"],
+            registry
+        )
+        .unwrap();
+
+        let over_window_cache_miss_count = register_int_counter_vec_with_registry!(
+            "stream_over_window_cache_miss_count",
+            "Over window executor cache miss count",
+            &["table_id", "actor_id"],
+            registry
+        )
+        .unwrap();
+
         let opts = histogram_opts!(
             "stream_barrier_inflight_duration_seconds",
             "barrier_inflight_latency",
@@ -603,6 +663,14 @@ impl StreamingMetrics {
             exponential_buckets(0.1, 1.5, 16).unwrap() // max 43s
         );
         let barrier_sync_latency = register_histogram_with_registry!(opts, registry).unwrap();
+
+        let barrier_manager_progress = register_int_counter_with_registry!(
+            "stream_barrier_manager_progress",
+            "The number of actors that have processed the earliest in-flight barriers",
+            registry
+        )
+        .unwrap();
+
         let sink_commit_duration = register_histogram_vec_with_registry!(
             "sink_commit_duration",
             "Duration of commit op in sink",
@@ -669,6 +737,20 @@ impl StreamingMetrics {
         )
         .unwrap();
 
+        let user_source_reader_error_count = register_int_counter_vec_with_registry!(
+            "user_source_reader_error_count",
+            "Source reader error count",
+            &[
+                "error_type",
+                "error_msg",
+                "executor_name",
+                "actor_id",
+                "source_id"
+            ],
+            registry,
+        )
+        .unwrap();
+
         let materialize_cache_hit_count = register_int_counter_vec_with_registry!(
             "stream_materialize_cache_hit_count",
             "Materialize executor cache hit count",
@@ -694,6 +776,7 @@ impl StreamingMetrics {
         .unwrap();
 
         Self {
+            level,
             executor_row_count,
             actor_execution_time,
             actor_output_buffer_blocking_duration_ns,
@@ -750,8 +833,12 @@ impl StreamingMetrics {
             backfill_upstream_output_row_count,
             arrangement_backfill_snapshot_read_row_count,
             arrangement_backfill_upstream_output_row_count,
+            over_window_cached_entry_count,
+            over_window_cache_lookup_count,
+            over_window_cache_miss_count,
             barrier_inflight_latency,
             barrier_sync_latency,
+            barrier_manager_progress,
             sink_commit_duration,
             lru_current_watermark_time_ms,
             lru_physical_now_ms,
@@ -761,6 +848,7 @@ impl StreamingMetrics {
             jemalloc_allocated_bytes,
             jemalloc_active_bytes,
             user_compute_error_count,
+            user_source_reader_error_count,
             materialize_cache_hit_count,
             materialize_cache_total_count,
             stream_memory_usage,
@@ -769,6 +857,6 @@ impl StreamingMetrics {
 
     /// Create a new `StreamingMetrics` instance used in tests or other places.
     pub fn unused() -> Self {
-        GLOBAL_STREAMING_METRICS.clone()
+        global_streaming_metrics(MetricLevel::Disabled)
     }
 }
diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs
index 397102a357211..d2cbf05d71f80 100644
--- a/src/stream/src/executor/now.rs
+++ b/src/stream/src/executor/now.rs
@@ -81,10 +81,10 @@ impl<S: StateStore> NowExecutor<S> {
                 state_table.init_epoch(barrier.epoch);
 
                 let state_row = {
-                    let data_iter = state_table.iter(Default::default()).await?;
+                    let data_iter = state_table.iter_row(Default::default()).await?;
                     pin_mut!(data_iter);
-                    if let Some(state_row) = data_iter.next().await {
-                        Some(state_row?)
+                    if let Some(keyed_row) = data_iter.next().await {
+                        Some(keyed_row?)
                     } else {
                         None
                     }
diff --git a/src/stream/src/executor/over_window/delta_btree_map.rs b/src/stream/src/executor/over_window/delta_btree_map.rs
index de8ef6a5082d1..4203eeb4958fc 100644
--- a/src/stream/src/executor/over_window/delta_btree_map.rs
+++ b/src/stream/src/executor/over_window/delta_btree_map.rs
@@ -18,11 +18,12 @@ use std::ops::Bound;
 
 use enum_as_inner::EnumAsInner;
 
-/// [`DeltaBTreeMap`] wraps a [`BTreeMap`] reference as a snapshot and an owned delta [`BTreeMap`],
+/// [`DeltaBTreeMap`] wraps two [`BTreeMap`] references respectively as snapshot and delta,
 /// providing cursor that can iterate over the updated version of the snapshot.
-pub(super) struct DeltaBTreeMap<'part, K: Ord, V> {
-    snapshot: &'part BTreeMap<K, V>,
-    delta: BTreeMap<K, Change<V>>,
+#[derive(Debug, Clone, Copy)]
+pub(super) struct DeltaBTreeMap<'a, K: Ord, V> {
+    snapshot: &'a BTreeMap<K, V>,
+    delta: &'a BTreeMap<K, Change<V>>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumAsInner)]
@@ -31,36 +32,36 @@ pub(super) enum Change<V> {
     Delete,
 }
 
-impl<'part, K: Ord, V> DeltaBTreeMap<'part, K, V> {
-    pub fn new(snapshot: &'part BTreeMap<K, V>, delta: BTreeMap<K, Change<V>>) -> Self {
+impl<'a, K: Ord, V> DeltaBTreeMap<'a, K, V> {
+    pub fn new(snapshot: &'a BTreeMap<K, V>, delta: &'a BTreeMap<K, Change<V>>) -> Self {
         Self { snapshot, delta }
     }
 
     /// Get a reference to the snapshot.
-    pub fn snapshot(&self) -> &'part BTreeMap<K, V> {
+    pub fn snapshot(&self) -> &'a BTreeMap<K, V> {
         self.snapshot
     }
 
     /// Get a reference to the delta.
-    pub fn delta(&self) -> &BTreeMap<K, Change<V>> {
-        &self.delta
+    pub fn delta(&self) -> &'a BTreeMap<K, Change<V>> {
+        self.delta
     }
 
     /// Get the first key in the updated version of the snapshot.
-    pub fn first_key(&self) -> Option<&K> {
+    pub fn first_key(&self) -> Option<&'a K> {
         let cursor = CursorWithDelta {
             snapshot: self.snapshot,
-            delta: &self.delta,
+            delta: self.delta,
             curr_key_value: None,
         };
         cursor.peek_next().map(|(key, _)| key)
     }
 
     /// Get the last key in the updated version of the snapshot.
-    pub fn last_key(&self) -> Option<&K> {
+    pub fn last_key(&self) -> Option<&'a K> {
         let cursor = CursorWithDelta {
             snapshot: self.snapshot,
-            delta: &self.delta,
+            delta: self.delta,
             curr_key_value: None,
         };
         cursor.peek_prev().map(|(key, _)| key)
@@ -68,7 +69,7 @@ impl<'part, K: Ord, V> DeltaBTreeMap<'part, K, V> {
 
     /// Get a [`CursorWithDelta`] pointing to the element corresponding to the given key.
     /// If the given key is not found in either the snapshot or the delta, `None` is returned.
-    pub fn find(&self, key: &K) -> Option<CursorWithDelta<'_, K, V>> {
+    pub fn find(&self, key: &K) -> Option<CursorWithDelta<'a, K, V>> {
         let ss_cursor = self.snapshot.lower_bound(Bound::Included(key));
         let dt_cursor = self.delta.lower_bound(Bound::Included(key));
         let curr_key_value = if dt_cursor.key() == Some(key) {
@@ -87,14 +88,14 @@ impl<'part, K: Ord, V> DeltaBTreeMap<'part, K, V> {
         };
         Some(CursorWithDelta {
             snapshot: self.snapshot,
-            delta: &self.delta,
+            delta: self.delta,
             curr_key_value: Some(curr_key_value),
         })
     }
 
-    /// Get a [`CursorWithDiff`] pointing to the first element that is above the given bound.
-    pub fn lower_bound(&self, bound: Bound<&K>) -> CursorWithDelta<'_, K, V> {
-        // the implementation is very similar to `CursorWithDiff::peek_next`
+    /// Get a [`CursorWithDelta`] pointing to the first element that is above the given bound.
+    pub fn lower_bound(&self, bound: Bound<&K>) -> CursorWithDelta<'a, K, V> {
+        // the implementation is very similar to `CursorWithDelta::peek_next`
         let mut ss_cursor = self.snapshot.lower_bound(bound);
         let mut dt_cursor = self.delta.lower_bound(bound);
         let next_ss_entry = || {
@@ -111,14 +112,14 @@ impl<'part, K: Ord, V> DeltaBTreeMap<'part, K, V> {
             CursorWithDelta::peek_impl(PeekDirection::Next, next_ss_entry, next_dt_entry);
         CursorWithDelta {
             snapshot: self.snapshot,
-            delta: &self.delta,
+            delta: self.delta,
             curr_key_value,
         }
     }
 
-    /// Get a [`CursorWithDiff`] pointing to the first element that is below the given bound.
-    pub fn upper_bound(&self, bound: Bound<&K>) -> CursorWithDelta<'_, K, V> {
-        // the implementation is very similar to `CursorWithDiff::peek_prev`
+    /// Get a [`CursorWithDelta`] pointing to the first element that is below the given bound.
+    pub fn upper_bound(&self, bound: Bound<&K>) -> CursorWithDelta<'a, K, V> {
+        // the implementation is very similar to `CursorWithDelta::peek_prev`
         let mut ss_cursor = self.snapshot.upper_bound(bound);
         let mut dt_cursor = self.delta.upper_bound(bound);
         let prev_ss_entry = || {
@@ -135,7 +136,7 @@ impl<'part, K: Ord, V> DeltaBTreeMap<'part, K, V> {
             CursorWithDelta::peek_impl(PeekDirection::Prev, prev_ss_entry, prev_dt_entry);
         CursorWithDelta {
             snapshot: self.snapshot,
-            delta: &self.delta,
+            delta: self.delta,
             curr_key_value,
         }
     }
@@ -321,7 +322,7 @@ mod tests {
     fn test_empty() {
         let map: BTreeMap<i32, &str> = BTreeMap::new();
         let delta = BTreeMap::new();
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), None);
         assert_eq!(delta_map.last_key(), None);
@@ -335,7 +336,7 @@ mod tests {
         let mut delta = BTreeMap::new();
         delta.insert(1, Change::Delete);
         delta.insert(2, Change::Delete);
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
         assert_eq!(delta_map.first_key(), None);
         assert_eq!(delta_map.last_key(), None);
         assert_eq!(delta_map.find(&1), None);
@@ -350,7 +351,7 @@ mod tests {
         map.insert(2, "2");
         map.insert(5, "5");
         let delta = BTreeMap::new();
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&5));
@@ -396,7 +397,7 @@ mod tests {
         let mut delta = BTreeMap::new();
         delta.insert(1, Change::Insert("1"));
         delta.insert(2, Change::Insert("2"));
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&2));
@@ -438,7 +439,7 @@ mod tests {
         map.insert(3, "3");
         let mut delta = BTreeMap::new();
         delta.insert(1, Change::Delete);
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&3));
         assert_eq!(delta_map.last_key(), Some(&3));
@@ -471,7 +472,7 @@ mod tests {
         map.insert(3, "3");
         let mut delta = BTreeMap::new();
         delta.insert(3, Change::Delete);
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&1));
@@ -494,7 +495,7 @@ mod tests {
         let mut delta = BTreeMap::new();
         delta.insert(1, Change::Delete);
         delta.insert(3, Change::Delete);
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), None);
         assert_eq!(delta_map.last_key(), None);
@@ -512,7 +513,7 @@ mod tests {
         map.insert(3, "3");
         let mut delta = BTreeMap::new();
         delta.insert(2, Change::Insert("2"));
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&3));
@@ -541,7 +542,7 @@ mod tests {
         map.insert(3, "3");
         let mut delta = BTreeMap::new();
         delta.insert(1, Change::Insert("1 new"));
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&3));
@@ -568,7 +569,7 @@ mod tests {
         map.insert(3, "3");
         let mut delta = BTreeMap::new();
         delta.insert(3, Change::Insert("3 new"));
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&3));
@@ -599,7 +600,7 @@ mod tests {
         delta.insert(1, Change::Insert("1 new"));
         delta.insert(3, Change::Delete);
         delta.insert(4, Change::Insert("4"));
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&0));
         assert_eq!(delta_map.last_key(), Some(&4));
@@ -663,7 +664,7 @@ mod tests {
         delta.insert(5, Change::Delete);
         delta.insert(7, Change::Delete);
         delta.insert(9, Change::Delete);
-        let delta_map = DeltaBTreeMap::new(&map, delta);
+        let delta_map = DeltaBTreeMap::new(&map, &delta);
 
         assert_eq!(delta_map.first_key(), Some(&0));
         assert_eq!(delta_map.last_key(), Some(&3));
diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs
index ccfc4bfabde05..22553641369c1 100644
--- a/src/stream/src/executor/over_window/eowc.rs
+++ b/src/stream/src/executor/over_window/eowc.rs
@@ -71,7 +71,7 @@ type PartitionCache = ManagedLruCache<MemcmpEncoded, Partition>; // TODO(rc): us
 ///
 /// Basic idea:
 ///
-/// ```ignore
+/// ```text
 /// ──────────────┬────────────────────────────────────────────────────── curr evict row
 ///               │ROWS BETWEEN 5 PRECEDING AND 1 PRECEDING
 ///        (1)    │ ─┬─
@@ -198,12 +198,12 @@ impl<S: StateStore> EowcOverWindowExecutor<S> {
         // Recover states from state table.
         let table_iter = this
             .state_table
-            .iter_with_pk_prefix(partition_key, PrefetchOptions::new_for_exhaust_iter())
+            .iter_row_with_pk_prefix(partition_key, PrefetchOptions::new_for_exhaust_iter())
             .await?;
 
         #[for_await]
-        for row in table_iter {
-            let row: OwnedRow = row?;
+        for keyed_row in table_iter {
+            let row = keyed_row?.into_owned_row();
             let order_key_enc = memcmp_encoding::encode_row(
                 row::once(Some(
                     row.datum_at(this.order_key_index)
diff --git a/src/stream/src/executor/over_window/estimated_btree_map.rs b/src/stream/src/executor/over_window/estimated_btree_map.rs
new file mode 100644
index 0000000000000..953fc66346db7
--- /dev/null
+++ b/src/stream/src/executor/over_window/estimated_btree_map.rs
@@ -0,0 +1,159 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+use std::ops::{Bound, RangeInclusive};
+
+use risingwave_common::estimate_size::{EstimateSize, KvSize};
+
+pub struct EstimatedBTreeMap<K, V> {
+    inner: BTreeMap<K, V>,
+    heap_size: KvSize,
+}
+
+impl<K, V> EstimatedBTreeMap<K, V> {
+    pub fn new() -> Self {
+        Self {
+            inner: BTreeMap::new(),
+            heap_size: KvSize::new(),
+        }
+    }
+
+    pub fn inner(&self) -> &BTreeMap<K, V> {
+        &self.inner
+    }
+
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+}
+
+impl<K, V> Default for EstimatedBTreeMap<K, V> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<K, V> EstimatedBTreeMap<K, V>
+where
+    K: EstimateSize + Ord,
+    V: EstimateSize,
+{
+    pub fn first_key_value(&self) -> Option<(&K, &V)> {
+        self.inner.first_key_value()
+    }
+
+    pub fn last_key_value(&self) -> Option<(&K, &V)> {
+        self.inner.last_key_value()
+    }
+
+    pub fn insert(&mut self, key: K, row: V) {
+        let key_size = self.heap_size.add_val(&key);
+        self.heap_size.add_val(&row);
+        if let Some(old_row) = self.inner.insert(key, row) {
+            self.heap_size.sub_size(key_size);
+            self.heap_size.sub_val(&old_row);
+        }
+    }
+
+    pub fn remove(&mut self, key: &K) {
+        if let Some(row) = self.inner.remove(key) {
+            self.heap_size.sub(key, &row);
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn clear(&mut self) {
+        self.inner.clear();
+        self.heap_size.set(0);
+    }
+
+    /// Retain the given range of entries in the map, removing others.
+    pub fn retain_range(&mut self, range: RangeInclusive<&K>) -> (BTreeMap<K, V>, BTreeMap<K, V>)
+    where
+        K: Clone,
+    {
+        let start = *range.start();
+        let end = *range.end();
+
+        // [ left, [mid], right ]
+        let mut mid_right = self.inner.split_off(start);
+        let mid_right_split_key = mid_right.lower_bound(Bound::Excluded(end)).key().cloned();
+        let right = if let Some(ref mid_right_split_key) = mid_right_split_key {
+            mid_right.split_off(mid_right_split_key)
+        } else {
+            Default::default()
+        };
+        let mid = mid_right;
+        let left = std::mem::replace(&mut self.inner, mid);
+
+        for (k, v) in &left {
+            self.heap_size.sub(k, v);
+        }
+        for (k, v) in &right {
+            self.heap_size.sub(k, v);
+        }
+
+        (left, right)
+    }
+}
+
+impl<K, V> EstimateSize for EstimatedBTreeMap<K, V>
+where
+    K: EstimateSize,
+    V: EstimateSize,
+{
+    fn estimated_heap_size(&self) -> usize {
+        self.heap_size.size()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::EstimatedBTreeMap;
+
+    #[test]
+    fn test_retain_range() {
+        let mut map = EstimatedBTreeMap::new();
+
+        let (left, right) = map.retain_range(&1..=&10);
+        assert!(left.is_empty());
+        assert!(right.is_empty());
+
+        map.insert(1, "hello".to_string());
+        map.insert(6, "world".to_string());
+        let (left, right) = map.retain_range(&6..=&6);
+        assert_eq!(map.len(), 1);
+        assert_eq!(map.inner[&6], "world".to_string());
+        assert_eq!(left.len(), 1);
+        assert_eq!(left[&1], "hello".to_string());
+        assert!(right.is_empty());
+
+        map.insert(8, "risingwave".to_string());
+        map.insert(3, "great".to_string());
+        map.insert(0, "wooow".to_string());
+        let (left, right) = map.retain_range(&2..=&7);
+        assert_eq!(map.len(), 2);
+        assert_eq!(map.inner[&3], "great".to_string());
+        assert_eq!(map.inner[&6], "world".to_string());
+        assert_eq!(left.len(), 1);
+        assert_eq!(left[&0], "wooow".to_string());
+        assert_eq!(right.len(), 1);
+        assert_eq!(right[&8], "risingwave".to_string());
+    }
+}
diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs
index bdb36323b7763..ba995f7e7c3cb 100644
--- a/src/stream/src/executor/over_window/general.rs
+++ b/src/stream/src/executor/over_window/general.rs
@@ -12,32 +12,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{btree_map, BTreeMap, HashSet};
 use std::marker::PhantomData;
-use std::ops::Bound;
+use std::ops::RangeInclusive;
+use std::sync::Arc;
 
 use futures::StreamExt;
-use futures_async_stream::{for_await, try_stream};
+use futures_async_stream::try_stream;
 use itertools::Itertools;
 use risingwave_common::array::stream_record::Record;
 use risingwave_common::array::{RowRef, StreamChunk};
 use risingwave_common::catalog::Field;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
-use risingwave_common::types::DefaultOrdered;
+use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy;
+use risingwave_common::types::{DataType, DefaultOrdered};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::memcmp_encoding::{self, MemcmpEncoded};
 use risingwave_common::util::sort_util::OrderType;
 use risingwave_expr::window_function::{
-    create_window_state, FrameBounds, StateKey, WindowFuncCall, WindowStates,
+    create_window_state, StateKey, WindowFuncCall, WindowStates,
 };
-use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::StateStore;
 
-use self::private::Partition;
-use super::delta_btree_map::{Change, DeltaBTreeMap};
+use super::delta_btree_map::Change;
+use super::over_partition::{
+    new_empty_partition_cache, shrink_partition_cache, CacheKey, OverPartition, PartitionCache,
+    PartitionDelta,
+};
 use crate::cache::{new_unbounded, ManagedLruCache};
 use crate::common::metrics::MetricsInfo;
-use crate::executor::aggregation::ChunkBuilder;
+use crate::common::StreamChunkBuilder;
+use crate::executor::monitor::StreamingMetrics;
 use crate::executor::over_window::delta_btree_map::PositionType;
 use crate::executor::test_utils::prelude::StateTable;
 use crate::executor::{
@@ -46,62 +51,6 @@ use crate::executor::{
 };
 use crate::task::AtomicU64Ref;
 
-mod private {
-    use std::collections::BTreeMap;
-
-    use risingwave_common::estimate_size::{EstimateSize, KvSize};
-    use risingwave_common::row::OwnedRow;
-    use risingwave_expr::window_function::StateKey;
-
-    pub(super) struct Partition {
-        /// Fully synced table cache for the partition. `StateKey (order key, input pk)` -> table
-        /// row.
-        cache: BTreeMap<StateKey, OwnedRow>,
-        heap_size: KvSize,
-    }
-
-    impl Partition {
-        pub fn new(cache: BTreeMap<StateKey, OwnedRow>) -> Self {
-            let heap_size = cache.iter().fold(KvSize::new(), |mut x, (k, v)| {
-                x.add(k, v);
-                x
-            });
-            Self { cache, heap_size }
-        }
-
-        pub fn cache(&self) -> &BTreeMap<StateKey, OwnedRow> {
-            &self.cache
-        }
-
-        pub fn insert(&mut self, key: StateKey, row: OwnedRow) {
-            let key_size = self.heap_size.add_val(&key);
-            self.heap_size.add_val(&row);
-            if let Some(old_row) = self.cache.insert(key, row) {
-                self.heap_size.sub_size(key_size);
-                self.heap_size.sub_val(&old_row);
-            }
-        }
-
-        pub fn remove(&mut self, key: &StateKey) {
-            if let Some(row) = self.cache.remove(key) {
-                self.heap_size.sub(key, &row);
-            }
-        }
-    }
-
-    impl EstimateSize for Partition {
-        fn estimated_heap_size(&self) -> usize {
-            self.heap_size.size()
-        }
-    }
-}
-
-/// Changes happened in one partition in the chunk. `StateKey (order key, input pk)` => `Change`.
-type Delta = BTreeMap<StateKey, Change<OwnedRow>>;
-
-/// `partition key` => `Partition`.
-type PartitionCache = ManagedLruCache<OwnedRow, Partition>;
-
 /// [`OverWindowExecutor`] consumes retractable input stream and produces window function outputs.
 /// One [`OverWindowExecutor`] can handle one combination of partition key and order key.
 ///
@@ -119,22 +68,35 @@ struct ExecutorInner<S: StateStore> {
     calls: Vec<WindowFuncCall>,
     partition_key_indices: Vec<usize>,
     order_key_indices: Vec<usize>,
+    order_key_data_types: Vec<DataType>,
     order_key_order_types: Vec<OrderType>,
     input_pk_indices: Vec<usize>,
     input_schema_len: usize,
 
     state_table: StateTable<S>,
     watermark_epoch: AtomicU64Ref,
+    metrics: Arc<StreamingMetrics>,
 
     /// The maximum size of the chunk produced by executor at a time.
     chunk_size: usize,
+    cache_policy: CachePolicy,
 }
 
 struct ExecutionVars<S: StateStore> {
-    partitions: PartitionCache,
+    /// partition key => partition range cache.
+    cached_partitions: ManagedLruCache<OwnedRow, PartitionCache>,
+    /// partition key => recently accessed range.
+    recently_accessed_ranges: BTreeMap<DefaultOrdered<OwnedRow>, RangeInclusive<StateKey>>,
+    stats: ExecutionStats,
     _phantom: PhantomData<S>,
 }
 
+#[derive(Default)]
+struct ExecutionStats {
+    cache_miss: u64,
+    cache_lookup: u64,
+}
+
 impl<S: StateStore> Executor for OverWindowExecutor<S> {
     fn execute(self: Box<Self>) -> crate::executor::BoxedMessageStream {
         self.executor_inner().boxed()
@@ -172,11 +134,11 @@ impl<S: StateStore> ExecutorInner<S> {
         )?)
     }
 
-    fn row_to_state_key(&self, full_row: impl Row + Copy) -> StreamExecutorResult<StateKey> {
-        Ok(StateKey {
+    fn row_to_cache_key(&self, full_row: impl Row + Copy) -> StreamExecutorResult<CacheKey> {
+        Ok(CacheKey::Normal(StateKey {
             order_key: self.encode_order_key(full_row)?,
             pk: self.get_input_pk(full_row).into(),
-        })
+        }))
     }
 }
 
@@ -194,8 +156,10 @@ pub struct OverWindowExecutorArgs<S: StateStore> {
 
     pub state_table: StateTable<S>,
     pub watermark_epoch: AtomicU64Ref,
+    pub metrics: Arc<StreamingMetrics>,
 
     pub chunk_size: usize,
+    pub cache_policy: CachePolicy,
 }
 
 impl<S: StateStore> OverWindowExecutor<S> {
@@ -210,6 +174,21 @@ impl<S: StateStore> OverWindowExecutor<S> {
             schema
         };
 
+        let has_unbounded_frame = args.calls.iter().any(|call| call.frame.is_unbounded());
+        let cache_policy = if has_unbounded_frame {
+            // For unbounded frames, we finally need all entries of the partition in the cache,
+            // so for simplicity we just use full cache policy for these cases.
+            CachePolicy::Full
+        } else {
+            args.cache_policy
+        };
+
+        let order_key_data_types = args
+            .order_key_indices
+            .iter()
+            .map(|i| schema.fields()[*i].data_type.clone())
+            .collect();
+
         Self {
             input: args.input,
             inner: ExecutorInner {
@@ -222,41 +201,19 @@ impl<S: StateStore> OverWindowExecutor<S> {
                 calls: args.calls,
                 partition_key_indices: args.partition_key_indices,
                 order_key_indices: args.order_key_indices,
+                order_key_data_types,
                 order_key_order_types: args.order_key_order_types,
                 input_pk_indices: input_info.pk_indices,
                 input_schema_len: input_info.schema.len(),
                 state_table: args.state_table,
                 watermark_epoch: args.watermark_epoch,
+                metrics: args.metrics,
                 chunk_size: args.chunk_size,
+                cache_policy,
             },
         }
     }
 
-    async fn ensure_partition_in_cache(
-        this: &ExecutorInner<S>,
-        cache: &mut PartitionCache,
-        partition_key: &OwnedRow,
-    ) -> StreamExecutorResult<()> {
-        if cache.contains(partition_key) {
-            return Ok(());
-        }
-
-        let mut cache_for_partition = BTreeMap::new();
-        let table_iter = this
-            .state_table
-            .iter_with_pk_prefix(partition_key, PrefetchOptions::new_for_exhaust_iter())
-            .await?;
-
-        #[for_await]
-        for row in table_iter {
-            let row: OwnedRow = row?;
-            cache_for_partition.insert(this.row_to_state_key(&row)?, row);
-        }
-
-        cache.put(partition_key.clone(), Partition::new(cache_for_partition));
-        Ok(())
-    }
-
     /// Merge changes by input pk in the given chunk, return a change iterator which guarantees that
     /// each pk only appears once. This method also validates the consistency of the input
     /// chunk.
@@ -340,9 +297,9 @@ impl<S: StateStore> OverWindowExecutor<S> {
         vars: &'a mut ExecutionVars<S>,
         chunk: StreamChunk,
     ) {
-        // `partition key` => `Delta`.
-        let mut deltas: BTreeMap<DefaultOrdered<OwnedRow>, Delta> = BTreeMap::new();
-        // `input pk` of update records of which the `partition key` or `order key` is changed.
+        // partition key => changes happened in the partition.
+        let mut deltas: BTreeMap<DefaultOrdered<OwnedRow>, PartitionDelta> = BTreeMap::new();
+        // input pk of update records of which the order key is changed.
         let mut key_change_updated_pks = HashSet::new();
 
         // Collect changes for each partition.
@@ -350,31 +307,33 @@ impl<S: StateStore> OverWindowExecutor<S> {
             match record {
                 Record::Insert { new_row } => {
                     let part_key = this.get_partition_key(new_row).into();
-                    let part_delta = deltas.entry(part_key).or_default();
+                    let part_delta = deltas.entry(part_key).or_insert(PartitionDelta::new());
                     part_delta.insert(
-                        this.row_to_state_key(new_row)?,
+                        this.row_to_cache_key(new_row)?,
                         Change::Insert(new_row.into_owned_row()),
                     );
                 }
                 Record::Delete { old_row } => {
                     let part_key = this.get_partition_key(old_row).into();
-                    let part_delta = deltas.entry(part_key).or_default();
-                    part_delta.insert(this.row_to_state_key(old_row)?, Change::Delete);
+                    let part_delta = deltas.entry(part_key).or_insert(PartitionDelta::new());
+                    part_delta.insert(this.row_to_cache_key(old_row)?, Change::Delete);
                 }
                 Record::Update { old_row, new_row } => {
                     let old_part_key = this.get_partition_key(old_row).into();
                     let new_part_key = this.get_partition_key(new_row).into();
-                    let old_state_key = this.row_to_state_key(old_row)?;
-                    let new_state_key = this.row_to_state_key(new_row)?;
+                    let old_state_key = this.row_to_cache_key(old_row)?;
+                    let new_state_key = this.row_to_cache_key(new_row)?;
                     if old_part_key == new_part_key && old_state_key == new_state_key {
                         // not a key-change update
-                        let part_delta = deltas.entry(old_part_key).or_default();
+                        let part_delta =
+                            deltas.entry(old_part_key).or_insert(PartitionDelta::new());
                         part_delta.insert(old_state_key, Change::Insert(new_row.into_owned_row()));
                     } else if old_part_key == new_part_key {
                         // order-change update, split into delete + insert, will be merged after
                         // building changes
                         key_change_updated_pks.insert(this.get_input_pk(old_row));
-                        let part_delta = deltas.entry(old_part_key).or_default();
+                        let part_delta =
+                            deltas.entry(old_part_key).or_insert(PartitionDelta::new());
                         part_delta.insert(old_state_key, Change::Delete);
                         part_delta.insert(new_state_key, Change::Insert(new_row.into_owned_row()));
                     } else {
@@ -382,9 +341,11 @@ impl<S: StateStore> OverWindowExecutor<S> {
                         // NOTE(rc): Since we append partition key to logical pk, we can't merge the
                         // delete + insert back to update later.
                         // TODO: IMO this behavior is problematic. Deep discussion is needed.
-                        let old_part_delta = deltas.entry(old_part_key).or_default();
+                        let old_part_delta =
+                            deltas.entry(old_part_key).or_insert(PartitionDelta::new());
                         old_part_delta.insert(old_state_key, Change::Delete);
-                        let new_part_delta = deltas.entry(new_part_key).or_default();
+                        let new_part_delta =
+                            deltas.entry(new_part_key).or_insert(PartitionDelta::new());
                         new_part_delta
                             .insert(new_state_key, Change::Insert(new_row.into_owned_row()));
                     }
@@ -394,15 +355,33 @@ impl<S: StateStore> OverWindowExecutor<S> {
 
         // `input pk` => `Record`
         let mut key_change_update_buffer = BTreeMap::new();
-        let mut chunk_builder = ChunkBuilder::new(this.chunk_size, &this.info.schema.data_types());
+        let mut chunk_builder =
+            StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types());
 
         // Build final changes partition by partition.
         for (part_key, delta) in deltas {
-            Self::ensure_partition_in_cache(this, &mut vars.partitions, &part_key).await?;
-            let mut partition = vars.partitions.get_mut(&part_key).unwrap();
+            vars.stats.cache_lookup += 1;
+            if !vars.cached_partitions.contains(&part_key.0) {
+                vars.stats.cache_miss += 1;
+                vars.cached_partitions
+                    .put(part_key.0.clone(), new_empty_partition_cache());
+            }
+            let mut cache = vars.cached_partitions.get_mut(&part_key).unwrap();
+            let mut partition = OverPartition::new(
+                &part_key,
+                &mut cache,
+                this.cache_policy,
+                &this.calls,
+                &this.partition_key_indices,
+                &this.order_key_data_types,
+                &this.order_key_order_types,
+                &this.order_key_indices,
+                &this.input_pk_indices,
+            );
 
             // Build changes for current partition.
-            let part_changes = Self::build_changes_for_partition(this, &partition, delta)?;
+            let (part_changes, accessed_range) =
+                Self::build_changes_for_partition(this, &mut partition, delta).await?;
 
             for (key, record) in part_changes {
                 // Build chunk and yield if needed.
@@ -433,16 +412,27 @@ impl<S: StateStore> OverWindowExecutor<S> {
                     }
                 }
 
-                // Update state table and partition cache.
-                this.state_table.write_record(record.as_ref());
-                match record {
-                    Record::Insert { new_row } | Record::Update { new_row, .. } => {
-                        // If `Update`, the update is not a key-change update, so it's safe to just
-                        // replace the existing item in the cache.
-                        partition.insert(key, new_row);
+                // Apply the change record.
+                partition.write_record(&mut this.state_table, key, record);
+            }
+
+            // Update recently accessed range for later shrinking cache.
+            if !this.cache_policy.is_full() && let Some(accessed_range) = accessed_range {
+                match vars.recently_accessed_ranges.entry(part_key) {
+                    btree_map::Entry::Vacant(vacant) => {
+                        vacant.insert(accessed_range);
                     }
-                    Record::Delete { .. } => {
-                        partition.remove(&key);
+                    btree_map::Entry::Occupied(mut occupied) => {
+                        let recently_accessed_range = occupied.get_mut();
+                        let min_start = accessed_range
+                            .start()
+                            .min(recently_accessed_range.start())
+                            .clone();
+                        let max_end = accessed_range
+                            .end()
+                            .max(recently_accessed_range.end())
+                            .clone();
+                        *recently_accessed_range = min_start..=max_end;
                     }
                 }
             }
@@ -454,24 +444,33 @@ impl<S: StateStore> OverWindowExecutor<S> {
         }
     }
 
-    fn build_changes_for_partition(
+    async fn build_changes_for_partition(
         this: &ExecutorInner<S>,
-        partition: &Partition,
-        delta: Delta,
-    ) -> StreamExecutorResult<BTreeMap<StateKey, Record<OwnedRow>>> {
-        let snapshot = partition.cache();
-        let part_with_delta = DeltaBTreeMap::new(snapshot, delta);
-        let delta = part_with_delta.delta();
+        partition: &mut OverPartition<'_, S>,
+        delta: PartitionDelta,
+    ) -> StreamExecutorResult<(
+        BTreeMap<StateKey, Record<OwnedRow>>,
+        Option<RangeInclusive<StateKey>>,
+    )> {
         assert!(!delta.is_empty(), "if there's no delta, we won't be here");
 
         let mut part_changes = BTreeMap::new();
 
+        // Find affected ranges, this also ensures that all rows in the affected ranges are loaded
+        // into the cache.
+        let (part_with_delta, affected_ranges) = partition
+            .find_affected_ranges(&this.state_table, &delta)
+            .await?;
+
+        let snapshot = part_with_delta.snapshot();
+        let delta = part_with_delta.delta();
+
         // Generate delete changes first, because deletes are skipped during iteration over
         // `part_with_delta` in the next step.
         for (key, change) in delta {
             if change.is_delete() {
                 part_changes.insert(
-                    key.clone(),
+                    key.as_normal_expect().clone(),
                     Record::Delete {
                         old_row: snapshot.get(key).unwrap().clone(),
                     },
@@ -479,12 +478,33 @@ impl<S: StateStore> OverWindowExecutor<S> {
             }
         }
 
-        for (first_frame_start, first_curr_key, last_curr_key, last_frame_end) in
-            find_affected_ranges(&this.calls, &part_with_delta)
-        {
+        let mut accessed_range: Option<RangeInclusive<StateKey>> = None;
+
+        for (first_frame_start, first_curr_key, last_curr_key, last_frame_end) in affected_ranges {
             assert!(first_frame_start <= first_curr_key);
             assert!(first_curr_key <= last_curr_key);
             assert!(last_curr_key <= last_frame_end);
+            assert!(first_frame_start.is_normal());
+            assert!(first_curr_key.is_normal());
+            assert!(last_curr_key.is_normal());
+            assert!(last_frame_end.is_normal());
+
+            if let Some(accessed_range) = accessed_range.as_mut() {
+                let min_start = first_frame_start
+                    .as_normal_expect()
+                    .min(accessed_range.start())
+                    .clone();
+                let max_end = last_frame_end
+                    .as_normal_expect()
+                    .max(accessed_range.end())
+                    .clone();
+                *accessed_range = min_start..=max_end;
+            } else {
+                accessed_range = Some(
+                    first_frame_start.as_normal_expect().clone()
+                        ..=last_frame_end.as_normal_expect().clone(),
+                );
+            }
 
             let mut states =
                 WindowStates::new(this.calls.iter().map(create_window_state).try_collect()?);
@@ -492,7 +512,7 @@ impl<S: StateStore> OverWindowExecutor<S> {
             // Populate window states with the affected range of rows.
             {
                 let mut cursor = part_with_delta
-                    .find(&first_frame_start)
+                    .find(first_frame_start)
                     .expect("first frame start key must exist");
                 while {
                     let (key, row) = cursor
@@ -502,7 +522,7 @@ impl<S: StateStore> OverWindowExecutor<S> {
                     for (call, state) in this.calls.iter().zip_eq_fast(states.iter_mut()) {
                         // TODO(rc): batch appending
                         state.append(
-                            key.clone(),
+                            key.as_normal_expect().clone(),
                             row.project(call.args.val_indices())
                                 .into_owned_row()
                                 .as_inner()
@@ -511,17 +531,20 @@ impl<S: StateStore> OverWindowExecutor<S> {
                     }
                     cursor.move_next();
 
-                    key != &last_frame_end
+                    key != last_frame_end
                 } {}
             }
 
             // Slide to the first affected key. We can safely compare to `Some(first_curr_key)` here
             // because it must exist in the states, by the definition of affected range.
-            while states.curr_key() != Some(&first_curr_key) {
+            while states.curr_key() != Some(first_curr_key.as_normal_expect()) {
                 states.just_slide_forward();
             }
-            let mut curr_key_cursor = part_with_delta.find(&first_curr_key).unwrap();
-            assert_eq!(states.curr_key(), curr_key_cursor.key());
+            let mut curr_key_cursor = part_with_delta.find(first_curr_key).unwrap();
+            assert_eq!(
+                states.curr_key(),
+                curr_key_cursor.key().map(CacheKey::as_normal_expect)
+            );
 
             // Slide and generate changes.
             while {
@@ -544,23 +567,27 @@ impl<S: StateStore> OverWindowExecutor<S> {
                         // update
                         let old_row = snapshot.get(key).unwrap().clone();
                         if old_row != new_row {
-                            part_changes.insert(key.clone(), Record::Update { old_row, new_row });
+                            part_changes.insert(
+                                key.as_normal_expect().clone(),
+                                Record::Update { old_row, new_row },
+                            );
                         }
                     }
                     PositionType::DeltaInsert => {
                         // insert
-                        part_changes.insert(key.clone(), Record::Insert { new_row });
+                        part_changes
+                            .insert(key.as_normal_expect().clone(), Record::Insert { new_row });
                     }
                 }
 
                 states.just_slide_forward();
                 curr_key_cursor.move_next();
 
-                key != &last_curr_key
+                key != last_curr_key
             } {}
         }
 
-        Ok(part_changes)
+        Ok((part_changes, accessed_range))
     }
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
@@ -578,14 +605,16 @@ impl<S: StateStore> OverWindowExecutor<S> {
         );
 
         let mut vars = ExecutionVars {
-            partitions: new_unbounded(this.watermark_epoch.clone(), metrics_info),
+            cached_partitions: new_unbounded(this.watermark_epoch.clone(), metrics_info),
+            recently_accessed_ranges: Default::default(),
+            stats: Default::default(),
             _phantom: PhantomData::<S>,
         };
 
         let mut input = input.execute();
         let barrier = expect_first_barrier(&mut input).await?;
         this.state_table.init_epoch(barrier.epoch);
-        vars.partitions.update_epoch(barrier.epoch.curr);
+        vars.cached_partitions.update_epoch(barrier.epoch.curr);
 
         yield Message::Barrier(barrier);
 
@@ -606,442 +635,56 @@ impl<S: StateStore> OverWindowExecutor<S> {
                 }
                 Message::Barrier(barrier) => {
                     this.state_table.commit(barrier.epoch).await?;
-                    vars.partitions.evict();
+                    vars.cached_partitions.evict();
+
+                    {
+                        // update metrics
+                        let actor_id_str = this.actor_ctx.id.to_string();
+                        let table_id_str = this.state_table.table_id().to_string();
+                        this.metrics
+                            .over_window_cached_entry_count
+                            .with_label_values(&[&table_id_str, &actor_id_str])
+                            .set(vars.cached_partitions.len() as _);
+                        this.metrics
+                            .over_window_cache_lookup_count
+                            .with_label_values(&[&table_id_str, &actor_id_str])
+                            .inc_by(std::mem::take(&mut vars.stats.cache_lookup));
+                        this.metrics
+                            .over_window_cache_miss_count
+                            .with_label_values(&[&table_id_str, &actor_id_str])
+                            .inc_by(std::mem::take(&mut vars.stats.cache_miss));
+                    }
 
                     if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(this.actor_ctx.id) {
                         let (_, cache_may_stale) =
                             this.state_table.update_vnode_bitmap(vnode_bitmap);
                         if cache_may_stale {
-                            vars.partitions.clear();
+                            vars.cached_partitions.clear();
                         }
                     }
 
-                    vars.partitions.update_epoch(barrier.epoch.curr);
-
-                    yield Message::Barrier(barrier);
-                }
-            }
-        }
-    }
-}
-
-/// Find all affected ranges in the given partition with delta.
-///
-/// # Returns
-///
-/// `Vec<(first_frame_start, first_curr_key, last_curr_key, last_frame_end_incl)>`
-///
-/// Each affected range is a union of many small window frames affected by some adajcent
-/// keys in the delta.
-///
-/// Example:
-/// - frame 1: `rows between 2 preceding and current row`
-/// - frame 2: `rows between 1 preceding and 2 following`
-/// - partition: `[1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14]`
-/// - delta: `[3, 4, 15]`
-/// - affected ranges: `[(1, 1, 7, 9), (10, 12, 15, 15)]`
-///
-/// TODO(rc):
-/// Note that, since we assume input chunks have data locality on order key columns, we now only
-/// calculate one single affected range. So the affected ranges in the above example will be
-/// `(1, 1, 15, 15)`. Later we may optimize this.
-fn find_affected_ranges(
-    calls: &[WindowFuncCall],
-    part_with_delta: &DeltaBTreeMap<'_, StateKey, OwnedRow>,
-) -> Vec<(StateKey, StateKey, StateKey, StateKey)> {
-    let delta = part_with_delta.delta();
-
-    if part_with_delta.first_key().is_none() {
-        // all keys are deleted in the delta
-        return vec![];
-    }
-
-    if part_with_delta.snapshot().is_empty() {
-        // all existing keys are inserted in the delta
-        return vec![(
-            delta.first_key_value().unwrap().0.clone(),
-            delta.first_key_value().unwrap().0.clone(),
-            delta.last_key_value().unwrap().0.clone(),
-            delta.last_key_value().unwrap().0.clone(),
-        )];
-    }
-
-    let first_key = part_with_delta.first_key().unwrap();
-    let last_key = part_with_delta.last_key().unwrap();
-
-    let start_is_unbounded = calls
-        .iter()
-        .any(|call| call.frame.bounds.start_is_unbounded());
-    let end_is_unbounded = calls
-        .iter()
-        .any(|call| call.frame.bounds.end_is_unbounded());
-
-    let first_curr_key = if end_is_unbounded {
-        // If the frame end is unbounded, the frame corresponding to the first key is always
-        // affected.
-        first_key.clone()
-    } else {
-        calls
-            .iter()
-            .map(|call| match &call.frame.bounds {
-                FrameBounds::Rows(_start, end) => {
-                    let mut cursor = part_with_delta
-                        .lower_bound(Bound::Included(delta.first_key_value().unwrap().0));
-                    for _ in 0..end.n_following_rows().unwrap() {
-                        // Note that we have to move before check, to handle situation where the
-                        // cursor is at ghost position at first.
-                        cursor.move_prev();
-                        if cursor.position().is_ghost() {
-                            break;
-                        }
-                    }
-                    cursor.key().unwrap_or(first_key)
-                }
-            })
-            .min()
-            .expect("# of window function calls > 0")
-            .clone()
-    };
-
-    let first_frame_start = if start_is_unbounded {
-        // If the frame start is unbounded, the first key always need to be included in the affected
-        // range.
-        first_key.clone()
-    } else {
-        calls
-            .iter()
-            .map(|call| match &call.frame.bounds {
-                FrameBounds::Rows(start, _end) => {
-                    let mut cursor = part_with_delta.find(&first_curr_key).unwrap();
-                    for _ in 0..start.n_preceding_rows().unwrap() {
-                        cursor.move_prev();
-                        if cursor.position().is_ghost() {
-                            break;
-                        }
-                    }
-                    cursor.key().unwrap_or(first_key)
-                }
-            })
-            .min()
-            .expect("# of window function calls > 0")
-            .clone()
-    };
-
-    let last_curr_key = if start_is_unbounded {
-        last_key.clone()
-    } else {
-        calls
-            .iter()
-            .map(|call| match &call.frame.bounds {
-                FrameBounds::Rows(start, _end) => {
-                    let mut cursor = part_with_delta
-                        .upper_bound(Bound::Included(delta.last_key_value().unwrap().0));
-                    for _ in 0..start.n_preceding_rows().unwrap() {
-                        cursor.move_next();
-                        if cursor.position().is_ghost() {
-                            break;
-                        }
-                    }
-                    cursor.key().unwrap_or(last_key)
-                }
-            })
-            .max()
-            .expect("# of window function calls > 0")
-            .clone()
-    };
-
-    let last_frame_end = if end_is_unbounded {
-        last_key.clone()
-    } else {
-        calls
-            .iter()
-            .map(|call| match &call.frame.bounds {
-                FrameBounds::Rows(_start, end) => {
-                    let mut cursor = part_with_delta.find(&last_curr_key).unwrap();
-                    for _ in 0..end.n_following_rows().unwrap() {
-                        cursor.move_next();
-                        if cursor.position().is_ghost() {
-                            break;
+                    if !this.cache_policy.is_full() {
+                        for (part_key, recently_accessed_range) in
+                            std::mem::take(&mut vars.recently_accessed_ranges)
+                        {
+                            if let Some(mut range_cache) =
+                                vars.cached_partitions.get_mut(&part_key.0)
+                            {
+                                shrink_partition_cache(
+                                    &part_key.0,
+                                    &mut range_cache,
+                                    this.cache_policy,
+                                    recently_accessed_range,
+                                );
+                            }
                         }
                     }
-                    cursor.key().unwrap_or(last_key)
-                }
-            })
-            .max()
-            .expect("# of window function calls > 0")
-            .clone()
-    };
-
-    if first_curr_key > last_curr_key {
-        // all affected keys are deleted in the delta
-        return vec![];
-    }
-
-    vec![(
-        first_frame_start,
-        first_curr_key,
-        last_curr_key,
-        last_frame_end,
-    )]
-}
-
-#[cfg(test)]
-mod tests {
-    use risingwave_common::types::DataType;
-    use risingwave_expr::agg::{AggArgs, AggKind};
-    use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncKind};
-
-    use super::*;
-
-    #[test]
-    fn test_find_affected_ranges() {
-        fn create_call(frame: Frame) -> WindowFuncCall {
-            WindowFuncCall {
-                kind: WindowFuncKind::Aggregate(AggKind::Sum),
-                args: AggArgs::Unary(DataType::Int32, 0),
-                return_type: DataType::Int32,
-                frame,
-            }
-        }
-
-        macro_rules! create_snapshot {
-            ($( $pk:expr ),* $(,)?) => {
-                {
-                    #[allow(unused_mut)]
-                    let mut snapshot = BTreeMap::new();
-                    $(
-                        snapshot.insert(
-                            StateKey {
-                                // order key doesn't matter here
-                                order_key: vec![].into(),
-                                pk: OwnedRow::new(vec![Some($pk.into())]).into(),
-                            },
-                            // value row doesn't matter here
-                            OwnedRow::empty(),
-                        );
-                    )*
-                    snapshot
-                }
-            };
-        }
 
-        macro_rules! create_change {
-            (Delete) => {
-                Change::Delete
-            };
-            (Insert) => {
-                Change::Insert(OwnedRow::empty())
-            };
-        }
+                    vars.cached_partitions.update_epoch(barrier.epoch.curr);
 
-        macro_rules! create_delta {
-            ($(( $pk:expr, $change:ident )),* $(,)?) => {
-                {
-                    #[allow(unused_mut)]
-                    let mut delta = BTreeMap::new();
-                    $(
-                        delta.insert(
-                            StateKey {
-                                // order key doesn't matter here
-                                order_key: vec![].into(),
-                                pk: OwnedRow::new(vec![Some($pk.into())]).into(),
-                            },
-                            // value row doesn't matter here
-                            create_change!( $change ),
-                        );
-                    )*
-                    delta
+                    yield Message::Barrier(barrier);
                 }
-            };
-        }
-
-        {
-            // test all empty
-            let snapshot = create_snapshot!();
-            let delta = create_delta!();
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-            let calls = vec![create_call(Frame::rows(
-                FrameBound::Preceding(2),
-                FrameBound::Preceding(1),
-            ))];
-            assert!(find_affected_ranges(&calls, &part_with_delta).is_empty());
-        }
-
-        {
-            // test insert delta only
-            let snapshot = create_snapshot!();
-            let delta = create_delta!((1, Insert), (2, Insert), (3, Insert));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-            let calls = vec![create_call(Frame::rows(
-                FrameBound::Preceding(2),
-                FrameBound::Preceding(1),
-            ))];
-            let affected_ranges = find_affected_ranges(&calls, &part_with_delta);
-            assert_eq!(affected_ranges.len(), 1);
-            let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                affected_ranges.into_iter().next().unwrap();
-            assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(1.into())]));
-            assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(1.into())]));
-            assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(3.into())]));
-        }
-
-        {
-            // test simple
-            let snapshot = create_snapshot!(1, 2, 3, 4, 5, 6);
-            let delta = create_delta!((2, Insert), (3, Delete));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-
-            {
-                let calls = vec![create_call(Frame::rows(
-                    FrameBound::Preceding(2),
-                    FrameBound::Preceding(1),
-                ))];
-                let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                    find_affected_ranges(&calls, &part_with_delta)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(1.into())]));
-                assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(2.into())]));
-                assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(5.into())]));
-                assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(5.into())]));
-            }
-
-            {
-                let calls = vec![create_call(Frame::rows(
-                    FrameBound::Preceding(1),
-                    FrameBound::Following(2),
-                ))];
-                let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                    find_affected_ranges(&calls, &part_with_delta)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(1.into())]));
-                assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(1.into())]));
-                assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(4.into())]));
-                assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(6.into())]));
-            }
-
-            {
-                let calls = vec![create_call(Frame::rows(
-                    FrameBound::CurrentRow,
-                    FrameBound::Following(2),
-                ))];
-                let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                    find_affected_ranges(&calls, &part_with_delta)
-                        .into_iter()
-                        .next()
-                        .unwrap();
-                assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(1.into())]));
-                assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(1.into())]));
-                assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(2.into())]));
-                assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(5.into())]));
             }
         }
-
-        {
-            // test multiple calls
-            let snapshot = create_snapshot!(1, 2, 3, 4, 5, 6);
-            let delta = create_delta!((2, Insert), (3, Delete));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-
-            let calls = vec![
-                create_call(Frame::rows(
-                    FrameBound::Preceding(1),
-                    FrameBound::Preceding(1),
-                )),
-                create_call(Frame::rows(
-                    FrameBound::Following(1),
-                    FrameBound::Following(1),
-                )),
-            ];
-            let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                find_affected_ranges(&calls, &part_with_delta)
-                    .into_iter()
-                    .next()
-                    .unwrap();
-            assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(1.into())]));
-            assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(1.into())]));
-            assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(4.into())]));
-            assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(5.into())]));
-        }
-
-        {
-            // test lag corner case
-            let snapshot = create_snapshot!(1, 2, 3, 4, 5, 6);
-            let delta = create_delta!((1, Delete), (2, Delete), (3, Delete));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-
-            let calls = vec![create_call(Frame::rows(
-                FrameBound::Preceding(1),
-                FrameBound::Preceding(1),
-            ))];
-            let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                find_affected_ranges(&calls, &part_with_delta)
-                    .into_iter()
-                    .next()
-                    .unwrap();
-            assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(4.into())]));
-            assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(4.into())]));
-            assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(4.into())]));
-            assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(4.into())]));
-        }
-
-        {
-            // test lead corner case
-            let snapshot = create_snapshot!(1, 2, 3, 4, 5, 6);
-            let delta = create_delta!((4, Delete), (5, Delete), (6, Delete));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-
-            let calls = vec![create_call(Frame::rows(
-                FrameBound::Following(1),
-                FrameBound::Following(1),
-            ))];
-            let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                find_affected_ranges(&calls, &part_with_delta)
-                    .into_iter()
-                    .next()
-                    .unwrap();
-            assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(3.into())]));
-        }
-
-        {
-            // test lag/lead(x, 0) corner case
-            let snapshot = create_snapshot!(1, 2, 3, 4);
-            let delta = create_delta!((2, Delete), (3, Delete));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-
-            let calls = vec![create_call(Frame::rows(
-                FrameBound::CurrentRow,
-                FrameBound::CurrentRow,
-            ))];
-            assert!(find_affected_ranges(&calls, &part_with_delta).is_empty());
-        }
-
-        {
-            // test lag/lead(x, 0) corner case 2
-            let snapshot = create_snapshot!(1, 2, 3, 4, 5);
-            let delta = create_delta!((2, Delete), (3, Insert), (4, Delete));
-            let part_with_delta = DeltaBTreeMap::new(&snapshot, delta);
-
-            let calls = vec![create_call(Frame::rows(
-                FrameBound::CurrentRow,
-                FrameBound::CurrentRow,
-            ))];
-            let (first_frame_start, first_curr_key, last_curr_key, last_frame_end) =
-                find_affected_ranges(&calls, &part_with_delta)
-                    .into_iter()
-                    .next()
-                    .unwrap();
-            assert_eq!(first_frame_start.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(first_curr_key.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(last_curr_key.pk.0, OwnedRow::new(vec![Some(3.into())]));
-            assert_eq!(last_frame_end.pk.0, OwnedRow::new(vec![Some(3.into())]));
-        }
     }
 }
diff --git a/src/stream/src/executor/over_window/mod.rs b/src/stream/src/executor/over_window/mod.rs
index 7e12f78b3051b..7a41bc87da060 100644
--- a/src/stream/src/executor/over_window/mod.rs
+++ b/src/stream/src/executor/over_window/mod.rs
@@ -14,7 +14,10 @@
 
 mod delta_btree_map;
 mod eowc;
+mod estimated_btree_map;
 mod general;
+mod over_partition;
+mod sentinel;
 
 pub use eowc::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs};
 pub use general::{OverWindowExecutor, OverWindowExecutorArgs};
diff --git a/src/stream/src/executor/over_window/over_partition.rs b/src/stream/src/executor/over_window/over_partition.rs
new file mode 100644
index 0000000000000..28d7ce7d2ef99
--- /dev/null
+++ b/src/stream/src/executor/over_window/over_partition.rs
@@ -0,0 +1,1370 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Types and functions that store or manipulate state/cache inside one single over window
+//! partition.
+
+use std::collections::{BTreeMap, HashSet, VecDeque};
+use std::marker::PhantomData;
+use std::ops::{Bound, RangeInclusive};
+
+use futures::stream::select_all;
+use futures::{stream, StreamExt, TryStreamExt};
+use futures_async_stream::for_await;
+use risingwave_common::array::stream_record::Record;
+use risingwave_common::hash::VnodeBitmapExt;
+use risingwave_common::row::{OwnedRow, Row, RowExt};
+use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy;
+use risingwave_common::types::DataType;
+use risingwave_common::util::memcmp_encoding;
+use risingwave_common::util::sort_util::OrderType;
+use risingwave_expr::window_function::{FrameBounds, StateKey, WindowFuncCall};
+use risingwave_storage::store::PrefetchOptions;
+use risingwave_storage::table::merge_sort::merge_sort;
+use risingwave_storage::StateStore;
+
+use super::delta_btree_map::Change;
+use super::estimated_btree_map::EstimatedBTreeMap;
+use super::sentinel::KeyWithSentinel;
+use crate::executor::over_window::delta_btree_map::DeltaBTreeMap;
+use crate::executor::test_utils::prelude::StateTable;
+use crate::executor::StreamExecutorResult;
+
+pub(super) type CacheKey = KeyWithSentinel<StateKey>;
+
+/// Range cache for one over window partition.
+/// The cache entries can be:
+///
+/// - `(Normal)*`
+/// - `Smallest, (Normal)*, Largest`
+/// - `(Normal)+, Largest`
+/// - `Smallest, (Normal)+`
+///
+/// This means it's impossible to only have one sentinel in the cache without any normal entry,
+/// and, each of the two types of sentinel can only appear once. Also, since sentinels are either
+/// smallest or largest, they always appear at the beginning or the end of the cache.
+pub(super) type PartitionCache = EstimatedBTreeMap<CacheKey, OwnedRow>;
+
+/// Changes happened in one over window partition.
+pub(super) type PartitionDelta = BTreeMap<CacheKey, Change<OwnedRow>>;
+
+pub(super) fn new_empty_partition_cache() -> PartitionCache {
+    let mut cache = PartitionCache::new();
+    cache.insert(CacheKey::Smallest, OwnedRow::empty());
+    cache.insert(CacheKey::Largest, OwnedRow::empty());
+    cache
+}
+
+const MAGIC_CACHE_SIZE: usize = 1024;
+const MAGIC_JITTER_PREVENTION: usize = MAGIC_CACHE_SIZE / 8;
+
+pub(super) fn shrink_partition_cache(
+    this_partition_key: &OwnedRow,
+    range_cache: &mut PartitionCache,
+    cache_policy: CachePolicy,
+    recently_accessed_range: RangeInclusive<StateKey>,
+) {
+    tracing::debug!(
+        this_partition_key=?this_partition_key,
+        cache_policy=?cache_policy,
+        recently_accessed_range=?recently_accessed_range,
+        "find the range to retain in the range cache"
+    );
+
+    let (start, end) = match cache_policy {
+        CachePolicy::Full => {
+            // evict nothing if the policy is to cache full partition
+            return;
+        }
+        CachePolicy::Recent => {
+            let (sk_start, sk_end) = recently_accessed_range.into_inner();
+            let (ck_start, ck_end) = (CacheKey::from(sk_start), CacheKey::from(sk_end));
+
+            let mut cursor = range_cache.inner().upper_bound(Bound::Excluded(&ck_start));
+            for _ in 0..MAGIC_JITTER_PREVENTION {
+                if cursor.key().is_none() {
+                    break;
+                }
+                cursor.move_prev();
+            }
+            let start = cursor
+                .key()
+                .unwrap_or_else(|| range_cache.first_key_value().unwrap().0)
+                .clone();
+
+            let mut cursor = range_cache.inner().lower_bound(Bound::Excluded(&ck_end));
+            for _ in 0..MAGIC_JITTER_PREVENTION {
+                if cursor.key().is_none() {
+                    break;
+                }
+                cursor.move_next();
+            }
+            let end = cursor
+                .key()
+                .unwrap_or_else(|| range_cache.last_key_value().unwrap().0)
+                .clone();
+
+            (start, end)
+        }
+        CachePolicy::RecentFirstN => {
+            if range_cache.len() <= MAGIC_CACHE_SIZE {
+                // no need to evict if cache len <= N
+                return;
+            } else {
+                let (sk_start, _sk_end) = recently_accessed_range.into_inner();
+                let ck_start = CacheKey::from(sk_start);
+
+                let mut capacity_remain = MAGIC_CACHE_SIZE; // precision is not important here, code simplicity is first
+
+                let mut cursor = range_cache.inner().upper_bound(Bound::Excluded(&ck_start));
+                // go back for at most `MAGIC_JITTER_PREVENTION` entries
+                for _ in 0..MAGIC_JITTER_PREVENTION {
+                    if cursor.key().is_none() {
+                        break;
+                    }
+                    cursor.move_prev();
+                    capacity_remain -= 1;
+                }
+                let start = cursor
+                    .key()
+                    .unwrap_or_else(|| range_cache.first_key_value().unwrap().0)
+                    .clone();
+
+                let mut cursor = range_cache.inner().lower_bound(Bound::Included(&ck_start));
+                // go forward for at most `capacity_remain` entries
+                for _ in 0..capacity_remain {
+                    if cursor.key().is_none() {
+                        break;
+                    }
+                    cursor.move_next();
+                }
+                let end = cursor
+                    .key()
+                    .unwrap_or_else(|| range_cache.last_key_value().unwrap().0)
+                    .clone();
+
+                (start, end)
+            }
+        }
+        CachePolicy::RecentLastN => {
+            if range_cache.len() <= MAGIC_CACHE_SIZE {
+                // no need to evict if cache len <= N
+                return;
+            } else {
+                let (_sk_start, sk_end) = recently_accessed_range.into_inner();
+                let ck_end = CacheKey::from(sk_end);
+
+                let mut capacity_remain = MAGIC_CACHE_SIZE; // precision is not important here, code simplicity is first
+
+                let mut cursor = range_cache.inner().lower_bound(Bound::Excluded(&ck_end));
+                // go forward for at most `MAGIC_JITTER_PREVENTION` entries
+                for _ in 0..MAGIC_JITTER_PREVENTION {
+                    if cursor.key().is_none() {
+                        break;
+                    }
+                    cursor.move_next();
+                    capacity_remain -= 1;
+                }
+                let end = cursor
+                    .key()
+                    .unwrap_or_else(|| range_cache.last_key_value().unwrap().0)
+                    .clone();
+
+                let mut cursor = range_cache.inner().upper_bound(Bound::Included(&ck_end));
+                // go back for at most `capacity_remain` entries
+                for _ in 0..capacity_remain {
+                    if cursor.key().is_none() {
+                        break;
+                    }
+                    cursor.move_prev();
+                }
+                let start = cursor
+                    .key()
+                    .unwrap_or_else(|| range_cache.first_key_value().unwrap().0)
+                    .clone();
+
+                (start, end)
+            }
+        }
+    };
+
+    tracing::debug!(
+        this_partition_key=?this_partition_key,
+        retain_range=?(&start..=&end),
+        "retain range in the range cache"
+    );
+
+    let (left_removed, right_removed) = range_cache.retain_range(&start..=&end);
+    if range_cache.is_empty() {
+        if !left_removed.is_empty() || !right_removed.is_empty() {
+            range_cache.insert(CacheKey::Smallest, OwnedRow::empty());
+            range_cache.insert(CacheKey::Largest, OwnedRow::empty());
+        }
+    } else {
+        if !left_removed.is_empty() {
+            range_cache.insert(CacheKey::Smallest, OwnedRow::empty());
+        }
+        if !right_removed.is_empty() {
+            range_cache.insert(CacheKey::Largest, OwnedRow::empty());
+        }
+    }
+}
+
+/// A wrapper of [`PartitionCache`] that provides helper methods to manipulate the cache.
+/// By putting this type inside `private` module, we can avoid misuse of the internal fields and
+/// methods.
+pub(super) struct OverPartition<'a, S: StateStore> {
+    this_partition_key: &'a OwnedRow,
+    range_cache: &'a mut PartitionCache,
+    cache_policy: CachePolicy,
+
+    calls: &'a [WindowFuncCall],
+    partition_key_indices: &'a [usize],
+    order_key_data_types: &'a [DataType],
+    order_key_order_types: &'a [OrderType],
+    order_key_indices: &'a [usize],
+    input_pk_indices: &'a [usize],
+    state_key_to_table_pk_proj: Vec<usize>,
+    _phantom: PhantomData<S>,
+}
+
+const MAGIC_BATCH_SIZE: usize = 512;
+
+impl<'a, S: StateStore> OverPartition<'a, S> {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        this_partition_key: &'a OwnedRow,
+        cache: &'a mut PartitionCache,
+        cache_policy: CachePolicy,
+        calls: &'a [WindowFuncCall],
+        partition_key_indices: &'a [usize],
+        order_key_data_types: &'a [DataType],
+        order_key_order_types: &'a [OrderType],
+        order_key_indices: &'a [usize],
+        input_pk_indices: &'a [usize],
+    ) -> Self {
+        // TODO(rc): move the calculation to executor?
+        let mut projection = Vec::with_capacity(
+            partition_key_indices.len() + order_key_indices.len() + input_pk_indices.len(),
+        );
+        let mut col_dedup = HashSet::new();
+        for (proj_idx, key_idx) in partition_key_indices
+            .iter()
+            .chain(order_key_indices.iter())
+            .chain(input_pk_indices.iter())
+            .enumerate()
+        {
+            if col_dedup.insert(*key_idx) {
+                projection.push(proj_idx);
+            }
+        }
+        projection.shrink_to_fit();
+
+        Self {
+            this_partition_key,
+            range_cache: cache,
+            cache_policy,
+
+            calls,
+            partition_key_indices,
+            order_key_data_types,
+            order_key_order_types,
+            order_key_indices,
+            input_pk_indices,
+            state_key_to_table_pk_proj: projection,
+            _phantom: PhantomData,
+        }
+    }
+
+    /// Get the number of cached entries ignoring sentinels.
+    pub fn cache_real_len(&self) -> usize {
+        let len = self.range_cache.inner().len();
+        if len <= 1 {
+            debug_assert!(self
+                .range_cache
+                .inner()
+                .first_key_value()
+                .map(|(k, _)| k.is_normal())
+                .unwrap_or(true));
+            return len;
+        }
+        // len >= 2
+        let cache_inner = self.range_cache.inner();
+        let sentinels = [
+            // sentinels only appear at the beginning and/or the end
+            cache_inner.first_key_value().unwrap().0.is_sentinel(),
+            cache_inner.last_key_value().unwrap().0.is_sentinel(),
+        ];
+        len - sentinels.into_iter().filter(|x| *x).count()
+    }
+
+    fn cache_real_first_key(&self) -> Option<&StateKey> {
+        self.range_cache
+            .inner()
+            .iter()
+            .find(|(k, _)| k.is_normal())
+            .map(|(k, _)| k.as_normal_expect())
+    }
+
+    fn cache_real_last_key(&self) -> Option<&StateKey> {
+        self.range_cache
+            .inner()
+            .iter()
+            .rev()
+            .find(|(k, _)| k.is_normal())
+            .map(|(k, _)| k.as_normal_expect())
+    }
+
+    fn cache_left_is_sentinel(&self) -> bool {
+        self.range_cache
+            .first_key_value()
+            .map(|(k, _)| k.is_sentinel())
+            .unwrap_or(false)
+    }
+
+    fn cache_right_is_sentinel(&self) -> bool {
+        self.range_cache
+            .last_key_value()
+            .map(|(k, _)| k.is_sentinel())
+            .unwrap_or(false)
+    }
+
+    /// Write a change record to state table and cache.
+    /// This function must be called after finding affected ranges, which means the change records
+    /// should never exceed the cached range.
+    pub fn write_record(
+        &mut self,
+        table: &mut StateTable<S>,
+        key: StateKey,
+        record: Record<OwnedRow>,
+    ) {
+        table.write_record(record.as_ref());
+        match record {
+            Record::Insert { new_row } | Record::Update { new_row, .. } => {
+                self.range_cache.insert(CacheKey::from(key), new_row);
+            }
+            Record::Delete { .. } => {
+                self.range_cache.remove(&CacheKey::from(key));
+
+                if self.cache_real_len() == 0 && self.range_cache.len() == 1 {
+                    // only one sentinel remains, should insert the other
+                    self.range_cache
+                        .insert(CacheKey::Smallest, OwnedRow::empty());
+                    self.range_cache
+                        .insert(CacheKey::Largest, OwnedRow::empty());
+                }
+            }
+        }
+    }
+
+    /// Find all ranges in the partition that are affected by the given delta.
+    /// The returned ranges are guaranteed to be sorted and non-overlapping. All keys in the ranges
+    /// are guaranteed to be cached.
+    pub async fn find_affected_ranges<'s, 'cache>(
+        &'s mut self,
+        table: &'_ StateTable<S>,
+        delta: &'cache PartitionDelta,
+    ) -> StreamExecutorResult<(
+        DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
+        Vec<(
+            &'cache CacheKey,
+            &'cache CacheKey,
+            &'cache CacheKey,
+            &'cache CacheKey,
+        )>,
+    )>
+    where
+        's: 'cache,
+    {
+        let delta_first = delta.first_key_value().unwrap().0.as_normal_expect();
+        let delta_last = delta.last_key_value().unwrap().0.as_normal_expect();
+
+        if self.cache_policy.is_full() {
+            // ensure everything is in the cache
+            self.extend_cache_to_boundary(table).await?;
+        } else {
+            // ensure the cache covers all delta (if possible)
+            self.extend_cache_by_range(table, delta_first..=delta_last)
+                .await?;
+        }
+
+        loop {
+            // Terminateability: `extend_cache_leftward_by_n` and `extend_cache_rightward_by_n` keep
+            // pushing the cache to the boundary of current partition. In these two methods, when
+            // any side of boundary is reached, the sentinel key will be removed, so finally
+            // `self::find_affected_ranges` will return ranges without any sentinels.
+
+            let (left_reached_sentinel, right_reached_sentinel) = {
+                // SAFETY: Here we shortly borrow the range cache and turn the reference into a
+                // `'cache` one to bypass the borrow checker. This is safe because we only return
+                // the reference once we don't need to do any further mutation.
+                let cache_inner = unsafe { &*(self.range_cache.inner() as *const _) };
+                let ranges =
+                    self::find_affected_ranges(self.calls, DeltaBTreeMap::new(cache_inner, delta));
+
+                if ranges.is_empty() {
+                    // no ranges affected, we're done
+                    return Ok((DeltaBTreeMap::new(cache_inner, delta), ranges));
+                }
+
+                let left_reached_sentinel = ranges.first().unwrap().0.is_sentinel();
+                let right_reached_sentinel = ranges.last().unwrap().3.is_sentinel();
+
+                if !left_reached_sentinel && !right_reached_sentinel {
+                    // all affected ranges are already cached, we're done
+                    return Ok((DeltaBTreeMap::new(cache_inner, delta), ranges));
+                }
+
+                (left_reached_sentinel, right_reached_sentinel)
+            };
+
+            if left_reached_sentinel {
+                // TODO(rc): should count cache miss for this, and also the below
+                tracing::debug!(partition=?self.this_partition_key, "partition cache left extension triggered");
+                let left_most = self.cache_real_first_key().unwrap_or(delta_first).clone();
+                self.extend_cache_leftward_by_n(table, &left_most).await?;
+            }
+            if right_reached_sentinel {
+                tracing::debug!(partition=?self.this_partition_key, "partition cache right extension triggered");
+                let right_most = self.cache_real_last_key().unwrap_or(delta_last).clone();
+                self.extend_cache_rightward_by_n(table, &right_most).await?;
+            }
+            tracing::debug!(partition=?self.this_partition_key, "partition cache extended");
+        }
+    }
+
+    async fn extend_cache_to_boundary(
+        &mut self,
+        table: &StateTable<S>,
+    ) -> StreamExecutorResult<()> {
+        if self.cache_real_len() == self.range_cache.len() {
+            // no sentinel in the cache, meaning we already cached all entries of this partition
+            return Ok(());
+        }
+
+        tracing::debug!(partition=?self.this_partition_key, "loading the whole partition into cache");
+
+        let mut new_cache = PartitionCache::new(); // shouldn't use `new_empty_partition_cache` here because we don't want sentinels
+        let table_iter = table
+            .iter_row_with_pk_prefix(
+                self.this_partition_key,
+                PrefetchOptions::new_for_exhaust_iter(),
+            )
+            .await?;
+
+        #[for_await]
+        for row in table_iter {
+            let row: OwnedRow = row?.into_owned_row();
+            new_cache.insert(self.row_to_state_key(&row)?.into(), row);
+        }
+        *self.range_cache = new_cache;
+
+        Ok(())
+    }
+
+    /// Try to load the given range of entries from table into cache.
+    /// When the function returns, it's guaranteed that there's no entry in the table that is within
+    /// the given range but not in the cache.
+    async fn extend_cache_by_range(
+        &mut self,
+        table: &StateTable<S>,
+        range: RangeInclusive<&StateKey>,
+    ) -> StreamExecutorResult<()> {
+        if self.cache_real_len() == self.range_cache.len() {
+            // no sentinel in the cache, meaning we already cached all entries of this partition
+            return Ok(());
+        }
+        assert!(self.range_cache.len() >= 2);
+
+        let cache_real_first_key = self.cache_real_first_key();
+        let cache_real_last_key = self.cache_real_last_key();
+
+        if cache_real_first_key.is_some() && *range.end() < cache_real_first_key.unwrap()
+            || cache_real_last_key.is_some() && *range.start() > cache_real_last_key.unwrap()
+        {
+            // completely not overlapping, for the sake of simplicity, we re-init the cache
+            tracing::debug!(
+                partition=?self.this_partition_key,
+                cache_first=?cache_real_first_key,
+                cache_last=?cache_real_last_key,
+                range=?range,
+                "modified range is completely non-overlapping with the cached range, re-initializing the cache"
+            );
+            *self.range_cache = new_empty_partition_cache();
+        }
+
+        if self.cache_real_len() == 0 {
+            // no normal entry in the cache, just load the given range
+            let table_pk_range = (
+                Bound::Included(self.state_key_to_table_pk(range.start())?),
+                Bound::Included(self.state_key_to_table_pk(range.end())?),
+            );
+            tracing::debug!(
+                partition=?self.this_partition_key,
+                table_pk_range=?table_pk_range,
+                "cache is empty, just loading the given range"
+            );
+            return self
+                .extend_cache_by_range_inner(table, table_pk_range)
+                .await;
+        }
+
+        // get the first and last keys again, now we are guaranteed to have at least a normal key
+        let cache_real_first_key = self.cache_real_first_key().unwrap();
+        let cache_real_last_key = self.cache_real_last_key().unwrap();
+
+        if self.cache_left_is_sentinel() && *range.start() < cache_real_first_key {
+            // extend leftward only if there's smallest sentinel
+            let table_pk_range = (
+                Bound::Included(self.state_key_to_table_pk(range.start())?),
+                Bound::Excluded(self.state_key_to_table_pk(cache_real_first_key)?),
+            );
+            tracing::debug!(
+                partition=?self.this_partition_key,
+                table_pk_range=?table_pk_range,
+                "loading the left half of given range"
+            );
+            return self
+                .extend_cache_by_range_inner(table, table_pk_range)
+                .await;
+        }
+
+        if self.cache_right_is_sentinel() && *range.end() > cache_real_last_key {
+            // extend rightward only if there's largest sentinel
+            let table_pk_range = (
+                Bound::Excluded(self.state_key_to_table_pk(cache_real_last_key)?),
+                Bound::Included(self.state_key_to_table_pk(range.end())?),
+            );
+            tracing::debug!(
+                partition=?self.this_partition_key,
+                table_pk_range=?table_pk_range,
+                "loading the right half of given range"
+            );
+            return self
+                .extend_cache_by_range_inner(table, table_pk_range)
+                .await;
+        }
+
+        // TODO(rc): Uncomment the following to enable prefetching rows before the start of the
+        // range once we have STATE TABLE REVERSE ITERATOR.
+        // self.extend_cache_leftward_by_n(table, range.start()).await?;
+
+        // prefetch rows after the end of the range
+        self.extend_cache_rightward_by_n(table, range.end()).await
+    }
+
+    async fn extend_cache_by_range_inner(
+        &mut self,
+        table: &StateTable<S>,
+        table_pk_range: (Bound<impl Row>, Bound<impl Row>),
+    ) -> StreamExecutorResult<()> {
+        let streams = stream::iter(table.vnode_bitmap().iter_vnodes())
+            .map(|vnode| {
+                table.iter_row_with_pk_range(
+                    &table_pk_range,
+                    vnode,
+                    PrefetchOptions::new_for_exhaust_iter(),
+                )
+            })
+            .buffer_unordered(10)
+            .try_collect::<Vec<_>>()
+            .await?
+            .into_iter()
+            .map(Box::pin);
+
+        #[for_await]
+        for row in select_all(streams) {
+            let row: OwnedRow = row?.into_owned_row();
+            let key = self.row_to_state_key(&row)?;
+            self.range_cache.insert(CacheKey::from(key), row);
+        }
+
+        Ok(())
+    }
+
+    async fn extend_cache_leftward_by_n(
+        &mut self,
+        table: &StateTable<S>,
+        hint_key: &StateKey,
+    ) -> StreamExecutorResult<()> {
+        if self.cache_real_len() == self.range_cache.len() {
+            // no sentinel in the cache, meaning we already cached all entries of this partition
+            return Ok(());
+        }
+        assert!(self.range_cache.len() >= 2);
+
+        let left_second = {
+            let mut iter = self.range_cache.inner().iter();
+            let left_first = iter.next().unwrap().0;
+            if left_first.is_normal() {
+                // the leftside already reaches the beginning of this partition in the table
+                return Ok(());
+            }
+            iter.next().unwrap().0
+        };
+        let range_to_exclusive = match left_second {
+            CacheKey::Normal(smallest_in_cache) => smallest_in_cache,
+            CacheKey::Largest => hint_key, // no normal entry in the cache
+            _ => unreachable!(),
+        }
+        .clone();
+
+        self.extend_cache_leftward_by_n_inner(table, &range_to_exclusive)
+            .await?;
+
+        if self.cache_real_len() == 0 {
+            // Cache was empty, and extending leftward didn't add anything to the cache, but we
+            // can't just remove the smallest sentinel, we must also try extending rightward.
+            self.extend_cache_rightward_by_n_inner(table, hint_key)
+                .await?;
+            if self.cache_real_len() == 0 {
+                // still empty, meaning the table is empty
+                self.range_cache.remove(&CacheKey::Smallest);
+                self.range_cache.remove(&CacheKey::Largest);
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn extend_cache_leftward_by_n_inner(
+        &mut self,
+        table: &StateTable<S>,
+        range_to_exclusive: &StateKey,
+    ) -> StreamExecutorResult<()> {
+        let mut to_extend: VecDeque<OwnedRow> = VecDeque::with_capacity(MAGIC_BATCH_SIZE);
+        {
+            let pk_range = (
+                Bound::Included(self.this_partition_key.into_owned_row()),
+                Bound::Excluded(self.state_key_to_table_pk(range_to_exclusive)?),
+            );
+            let streams: Vec<_> =
+                futures::future::try_join_all(table.vnode_bitmap().iter_vnodes().map(|vnode| {
+                    table.iter_row_with_pk_range(
+                        &pk_range,
+                        vnode,
+                        PrefetchOptions::new_for_exhaust_iter(),
+                    )
+                }))
+                .await?
+                .into_iter()
+                .map(Box::pin)
+                .collect();
+
+            #[for_await]
+            for row in merge_sort(streams) {
+                let row: OwnedRow = row?.into_owned_row();
+
+                // For leftward extension, we now must iterate the table in order from the beginning
+                // of this partition and fill only the last n rows to the cache.
+                // TODO(rc): WE NEED STATE TABLE REVERSE ITERATOR!!
+                if to_extend.len() == MAGIC_BATCH_SIZE {
+                    to_extend.pop_front();
+                }
+                to_extend.push_back(row);
+            }
+        }
+
+        let n_extended = to_extend.len();
+        for row in to_extend {
+            let key = self.row_to_state_key(&row)?;
+            self.range_cache.insert(CacheKey::from(key), row);
+        }
+        if n_extended < MAGIC_BATCH_SIZE && self.cache_real_len() > 0 {
+            // we reached the beginning of this partition in the table
+            self.range_cache.remove(&CacheKey::Smallest);
+        }
+
+        Ok(())
+    }
+
+    async fn extend_cache_rightward_by_n(
+        &mut self,
+        table: &StateTable<S>,
+        hint_key: &StateKey,
+    ) -> StreamExecutorResult<()> {
+        if self.cache_real_len() == self.range_cache.len() {
+            // no sentinel in the cache, meaning we already cached all entries of this partition
+            return Ok(());
+        }
+        assert!(self.range_cache.len() >= 2);
+
+        let right_second = {
+            let mut iter = self.range_cache.inner().iter();
+            let right_first = iter.next_back().unwrap().0;
+            if right_first.is_normal() {
+                // the rightside already reaches the end of this partition in the table
+                return Ok(());
+            }
+            iter.next_back().unwrap().0
+        };
+        let range_from_exclusive = match right_second {
+            CacheKey::Normal(largest_in_cache) => largest_in_cache,
+            CacheKey::Smallest => hint_key, // no normal entry in the cache
+            _ => unreachable!(),
+        }
+        .clone();
+
+        self.extend_cache_rightward_by_n_inner(table, &range_from_exclusive)
+            .await?;
+
+        if self.cache_real_len() == 0 {
+            // Cache was empty, and extending rightward didn't add anything to the cache, but we
+            // can't just remove the smallest sentinel, we must also try extending leftward.
+            self.extend_cache_leftward_by_n_inner(table, hint_key)
+                .await?;
+            if self.cache_real_len() == 0 {
+                // still empty, meaning the table is empty
+                self.range_cache.remove(&CacheKey::Smallest);
+                self.range_cache.remove(&CacheKey::Largest);
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn extend_cache_rightward_by_n_inner(
+        &mut self,
+        table: &StateTable<S>,
+        range_from_exclusive: &StateKey,
+    ) -> StreamExecutorResult<()> {
+        let mut n_extended = 0usize;
+        {
+            let pk_range = (
+                Bound::Excluded(self.state_key_to_table_pk(range_from_exclusive)?),
+                // currently we can't get the first possible key after this partition, so use
+                // `Unbounded` plus manual check for workaround
+                Bound::<OwnedRow>::Unbounded,
+            );
+            let streams: Vec<_> =
+                futures::future::try_join_all(table.vnode_bitmap().iter_vnodes().map(|vnode| {
+                    table.iter_row_with_pk_range(&pk_range, vnode, PrefetchOptions::default())
+                }))
+                .await?
+                .into_iter()
+                .map(Box::pin)
+                .collect();
+
+            #[for_await]
+            for row in merge_sort(streams) {
+                let row: OwnedRow = row?.into_owned_row();
+
+                if !Row::eq(
+                    self.this_partition_key,
+                    (&row).project(self.partition_key_indices),
+                ) {
+                    // we've reached the end of this partition
+                    break;
+                }
+
+                let key = self.row_to_state_key(&row)?;
+                self.range_cache.insert(CacheKey::from(key), row);
+
+                n_extended += 1;
+                if n_extended == MAGIC_BATCH_SIZE {
+                    break;
+                }
+            }
+        }
+
+        if n_extended < MAGIC_BATCH_SIZE && self.cache_real_len() > 0 {
+            // we reached the end of this partition in the table
+            self.range_cache.remove(&CacheKey::Largest);
+        }
+
+        Ok(())
+    }
+
+    fn state_key_to_table_pk(&self, key: &StateKey) -> StreamExecutorResult<OwnedRow> {
+        Ok(self
+            .this_partition_key
+            .chain(memcmp_encoding::decode_row(
+                &key.order_key,
+                self.order_key_data_types,
+                self.order_key_order_types,
+            )?)
+            .chain(key.pk.as_inner())
+            .project(&self.state_key_to_table_pk_proj)
+            .into_owned_row())
+    }
+
+    fn row_to_state_key(&self, full_row: impl Row + Copy) -> StreamExecutorResult<StateKey> {
+        Ok(StateKey {
+            order_key: memcmp_encoding::encode_row(
+                full_row.project(self.order_key_indices),
+                self.order_key_order_types,
+            )?,
+            pk: full_row
+                .project(self.input_pk_indices)
+                .into_owned_row()
+                .into(),
+        })
+    }
+}
+
+/// Find all affected ranges in the given partition with delta.
+///
+/// # Returns
+///
+/// `Vec<(first_frame_start, first_curr_key, last_curr_key, last_frame_end_incl)>`
+///
+/// Each affected range is a union of many small window frames affected by some adajcent
+/// keys in the delta.
+///
+/// Example:
+/// - frame 1: `rows between 2 preceding and current row`
+/// - frame 2: `rows between 1 preceding and 2 following`
+/// - partition: `[1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14]`
+/// - delta: `[3, 4, 15]`
+/// - affected ranges: `[(1, 1, 7, 9), (10, 12, 15, 15)]`
+///
+/// TODO(rc):
+/// Note that, since we assume input chunks have data locality on order key columns, we now only
+/// calculate one single affected range. So the affected ranges in the above example will be
+/// `(1, 1, 15, 15)`. Later we may optimize this.
+fn find_affected_ranges<'cache>(
+    calls: &'_ [WindowFuncCall],
+    part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
+) -> Vec<(
+    &'cache CacheKey,
+    &'cache CacheKey,
+    &'cache CacheKey,
+    &'cache CacheKey,
+)> {
+    // XXX(rc): NOTE FOR DEVS
+    // Must carefully consider the sentinel keys in the cache when extending this function to
+    // support `RANGE` and `GROUPS` frames later. May introduce a return value variant to clearly
+    // tell the caller that there exists at least one affected range that touches the sentinel.
+
+    let delta = part_with_delta.delta();
+
+    if part_with_delta.first_key().is_none() {
+        // all keys are deleted in the delta
+        return vec![];
+    }
+
+    if part_with_delta.snapshot().is_empty() {
+        // all existing keys are inserted in the delta
+        return vec![(
+            delta.first_key_value().unwrap().0,
+            delta.first_key_value().unwrap().0,
+            delta.last_key_value().unwrap().0,
+            delta.last_key_value().unwrap().0,
+        )];
+    }
+
+    let first_key = part_with_delta.first_key().unwrap();
+    let last_key = part_with_delta.last_key().unwrap();
+
+    let start_is_unbounded = calls
+        .iter()
+        .any(|call| call.frame.bounds.start_is_unbounded());
+    let end_is_unbounded = calls
+        .iter()
+        .any(|call| call.frame.bounds.end_is_unbounded());
+
+    let first_curr_key = if end_is_unbounded {
+        // If the frame end is unbounded, the frame corresponding to the first key is always
+        // affected.
+        first_key
+    } else {
+        calls
+            .iter()
+            .map(|call| match &call.frame.bounds {
+                FrameBounds::Rows(_start, end) => {
+                    let mut cursor = part_with_delta
+                        .lower_bound(Bound::Included(delta.first_key_value().unwrap().0));
+                    for _ in 0..end.n_following_rows().unwrap() {
+                        // Note that we have to move before check, to handle situation where the
+                        // cursor is at ghost position at first.
+                        cursor.move_prev();
+                        if cursor.position().is_ghost() {
+                            break;
+                        }
+                    }
+                    cursor.key().unwrap_or(first_key)
+                }
+            })
+            .min()
+            .expect("# of window function calls > 0")
+    };
+
+    let first_frame_start = if start_is_unbounded {
+        // If the frame start is unbounded, the first key always need to be included in the affected
+        // range.
+        first_key
+    } else {
+        calls
+            .iter()
+            .map(|call| match &call.frame.bounds {
+                FrameBounds::Rows(start, _end) => {
+                    let mut cursor = part_with_delta.find(first_curr_key).unwrap();
+                    for _ in 0..start.n_preceding_rows().unwrap() {
+                        cursor.move_prev();
+                        if cursor.position().is_ghost() {
+                            break;
+                        }
+                    }
+                    cursor.key().unwrap_or(first_key)
+                }
+            })
+            .min()
+            .expect("# of window function calls > 0")
+    };
+
+    let last_curr_key = if start_is_unbounded {
+        last_key
+    } else {
+        calls
+            .iter()
+            .map(|call| match &call.frame.bounds {
+                FrameBounds::Rows(start, _end) => {
+                    let mut cursor = part_with_delta
+                        .upper_bound(Bound::Included(delta.last_key_value().unwrap().0));
+                    for _ in 0..start.n_preceding_rows().unwrap() {
+                        cursor.move_next();
+                        if cursor.position().is_ghost() {
+                            break;
+                        }
+                    }
+                    cursor.key().unwrap_or(last_key)
+                }
+            })
+            .max()
+            .expect("# of window function calls > 0")
+    };
+
+    let last_frame_end = if end_is_unbounded {
+        last_key
+    } else {
+        calls
+            .iter()
+            .map(|call| match &call.frame.bounds {
+                FrameBounds::Rows(_start, end) => {
+                    let mut cursor = part_with_delta.find(last_curr_key).unwrap();
+                    for _ in 0..end.n_following_rows().unwrap() {
+                        cursor.move_next();
+                        if cursor.position().is_ghost() {
+                            break;
+                        }
+                    }
+                    cursor.key().unwrap_or(last_key)
+                }
+            })
+            .max()
+            .expect("# of window function calls > 0")
+    };
+
+    if first_curr_key > last_curr_key {
+        // all affected keys are deleted in the delta
+        return vec![];
+    }
+
+    vec![(
+        first_frame_start,
+        first_curr_key,
+        last_curr_key,
+        last_frame_end,
+    )]
+}
+
+#[cfg(test)]
+mod find_affected_ranges_tests {
+    //! Function `find_affected_ranges` is important enough to deserve its own test module. We must
+    //! test it thoroughly.
+
+    use itertools::Itertools;
+    use risingwave_common::types::{DataType, ScalarImpl};
+    use risingwave_expr::agg::{AggArgs, AggKind};
+    use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncKind};
+
+    use super::*;
+
+    fn create_call(frame: Frame) -> WindowFuncCall {
+        WindowFuncCall {
+            kind: WindowFuncKind::Aggregate(AggKind::Sum),
+            args: AggArgs::Unary(DataType::Int32, 0),
+            return_type: DataType::Int32,
+            frame,
+        }
+    }
+
+    macro_rules! create_cache {
+        (..., $( $pk:literal ),* , ...) => {
+            {
+                let mut cache = create_cache!( $( $pk ),* );
+                cache.insert(CacheKey::Smallest, OwnedRow::empty().into());
+                cache.insert(CacheKey::Largest, OwnedRow::empty().into());
+                cache
+            }
+        };
+        (..., $( $pk:literal ),*) => {
+            {
+                let mut cache = create_cache!( $( $pk ),* );
+                cache.insert(CacheKey::Smallest, OwnedRow::empty().into());
+                cache
+            }
+        };
+        ($( $pk:literal ),* , ...) => {
+            {
+                let mut cache = create_cache!( $( $pk ),* );
+                cache.insert(CacheKey::Largest, OwnedRow::empty().into());
+                cache
+            }
+        };
+        ($( $pk:literal ),*) => {
+            {
+                #[allow(unused_mut)]
+                let mut cache = BTreeMap::new();
+                $(
+                    cache.insert(
+                        CacheKey::Normal(
+                            StateKey {
+                                // order key doesn't matter here
+                                order_key: vec![].into(),
+                                pk: OwnedRow::new(vec![Some($pk.into())]).into(),
+                            },
+                        ),
+                        // value row doesn't matter here
+                        OwnedRow::empty(),
+                    );
+                )*
+                cache
+            }
+        };
+    }
+
+    macro_rules! create_change {
+        (Delete) => {
+            Change::Delete
+        };
+        (Insert) => {
+            Change::Insert(OwnedRow::empty())
+        };
+    }
+
+    macro_rules! create_delta {
+        ($(( $pk:literal, $change:ident )),* $(,)?) => {
+            {
+                #[allow(unused_mut)]
+                let mut delta = BTreeMap::new();
+                $(
+                    delta.insert(
+                        CacheKey::Normal(
+                            StateKey {
+                                // order key doesn't matter here
+                                order_key: vec![].into(),
+                                pk: OwnedRow::new(vec![Some($pk.into())]).into(),
+                            },
+                        ),
+                        // value row doesn't matter here
+                        create_change!( $change ),
+                    );
+                )*
+                delta
+            }
+        };
+    }
+
+    fn assert_ranges_eq(
+        result: Vec<(&CacheKey, &CacheKey, &CacheKey, &CacheKey)>,
+        expected: impl IntoIterator<Item = (ScalarImpl, ScalarImpl, ScalarImpl, ScalarImpl)>,
+    ) {
+        result
+            .into_iter()
+            .zip_eq(expected.into_iter())
+            .for_each(|(result, expected)| {
+                assert_eq!(
+                    result.0.as_normal_expect().pk.0,
+                    OwnedRow::new(vec![Some(expected.0)])
+                );
+                assert_eq!(
+                    result.1.as_normal_expect().pk.0,
+                    OwnedRow::new(vec![Some(expected.1)])
+                );
+                assert_eq!(
+                    result.2.as_normal_expect().pk.0,
+                    OwnedRow::new(vec![Some(expected.2)])
+                );
+                assert_eq!(
+                    result.3.as_normal_expect().pk.0,
+                    OwnedRow::new(vec![Some(expected.3)])
+                );
+            })
+    }
+
+    #[test]
+    fn test_all_empty() {
+        let cache = create_cache!();
+        let delta = create_delta!();
+        let calls = vec![create_call(Frame::rows(
+            FrameBound::Preceding(2),
+            FrameBound::Preceding(1),
+        ))];
+        assert_ranges_eq(
+            find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+            [],
+        );
+    }
+
+    #[test]
+    fn test_insert_delta_only() {
+        let cache = create_cache!();
+        let delta = create_delta!((1, Insert), (2, Insert), (3, Insert));
+        let calls = vec![create_call(Frame::rows(
+            FrameBound::Preceding(2),
+            FrameBound::Preceding(1),
+        ))];
+        let affected_ranges = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta));
+        assert_ranges_eq(affected_ranges, [(1.into(), 1.into(), 3.into(), 3.into())]);
+    }
+
+    #[test]
+    fn test_simple() {
+        let cache = create_cache!(1, 2, 3, 4, 5, 6);
+        let delta = create_delta!((2, Insert), (3, Delete));
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Preceding(2),
+                FrameBound::Preceding(1),
+            ))];
+            assert_ranges_eq(
+                find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+                [(1.into(), 2.into(), 5.into(), 5.into())],
+            );
+        }
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Preceding(1),
+                FrameBound::Following(2),
+            ))];
+            assert_ranges_eq(
+                find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+                [(1.into(), 1.into(), 4.into(), 6.into())],
+            );
+        }
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::CurrentRow,
+                FrameBound::Following(2),
+            ))];
+            assert_ranges_eq(
+                find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+                [(1.into(), 1.into(), 2.into(), 5.into())],
+            );
+        }
+    }
+
+    #[test]
+    fn test_multiple_calls() {
+        let cache = create_cache!(1, 2, 3, 4, 5, 6);
+        let delta = create_delta!((2, Insert), (3, Delete));
+        let calls = vec![
+            create_call(Frame::rows(
+                FrameBound::Preceding(1),
+                FrameBound::Preceding(1),
+            )),
+            create_call(Frame::rows(
+                FrameBound::Following(1),
+                FrameBound::Following(1),
+            )),
+        ];
+        assert_ranges_eq(
+            find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+            [(1.into(), 1.into(), 4.into(), 5.into())],
+        );
+    }
+
+    #[test]
+    fn test_lag_corner_case() {
+        let cache = create_cache!(1, 2, 3, 4, 5, 6);
+        let delta = create_delta!((1, Delete), (2, Delete), (3, Delete));
+        let calls = vec![create_call(Frame::rows(
+            FrameBound::Preceding(1),
+            FrameBound::Preceding(1),
+        ))];
+        assert_ranges_eq(
+            find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+            [(4.into(), 4.into(), 4.into(), 4.into())],
+        );
+    }
+
+    #[test]
+    fn test_lead_corner_case() {
+        let cache = create_cache!(1, 2, 3, 4, 5, 6);
+        let delta = create_delta!((4, Delete), (5, Delete), (6, Delete));
+        let calls = vec![create_call(Frame::rows(
+            FrameBound::Following(1),
+            FrameBound::Following(1),
+        ))];
+        assert_ranges_eq(
+            find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+            [(3.into(), 3.into(), 3.into(), 3.into())],
+        );
+    }
+
+    #[test]
+    fn test_lag_lead_offset_0_corner_case_1() {
+        let cache = create_cache!(1, 2, 3, 4);
+        let delta = create_delta!((2, Delete), (3, Delete));
+        let calls = vec![create_call(Frame::rows(
+            FrameBound::CurrentRow,
+            FrameBound::CurrentRow,
+        ))];
+        assert_ranges_eq(
+            find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+            [],
+        );
+    }
+
+    #[test]
+    fn test_lag_lead_offset_0_corner_case_2() {
+        let cache = create_cache!(1, 2, 3, 4, 5);
+        let delta = create_delta!((2, Delete), (3, Insert), (4, Delete));
+        let calls = vec![create_call(Frame::rows(
+            FrameBound::CurrentRow,
+            FrameBound::CurrentRow,
+        ))];
+        assert_ranges_eq(
+            find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+            [(3.into(), 3.into(), 3.into(), 3.into())],
+        );
+    }
+
+    #[test]
+    fn test_empty_with_sentinels() {
+        let cache: BTreeMap<KeyWithSentinel<StateKey>, OwnedRow> = create_cache!(..., , ...);
+        let delta = create_delta!((1, Insert), (2, Insert));
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::CurrentRow,
+                FrameBound::CurrentRow,
+            ))];
+            assert_ranges_eq(
+                find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+                [(1.into(), 1.into(), 2.into(), 2.into())],
+            );
+        }
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Preceding(1),
+                FrameBound::Preceding(1),
+            ))];
+            let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0];
+            assert!(range.0.is_smallest());
+            assert_eq!(
+                range.1.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(1.into())])
+            );
+            assert!(range.2.is_largest());
+            assert!(range.3.is_largest());
+        }
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Following(1),
+                FrameBound::Following(3),
+            ))];
+            let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0];
+            assert!(range.0.is_smallest());
+            assert!(range.1.is_smallest());
+            assert_eq!(
+                range.2.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(2.into())])
+            );
+            assert!(range.3.is_largest());
+        }
+    }
+
+    #[test]
+    fn test_with_left_sentinel() {
+        let cache = create_cache!(..., 2, 4, 5, 8);
+        let delta = create_delta!((3, Insert), (4, Insert), (8, Delete));
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Following(1),
+                FrameBound::Following(1),
+            ))];
+            assert_ranges_eq(
+                find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+                [(2.into(), 2.into(), 5.into(), 5.into())],
+            );
+        }
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Preceding(1),
+                FrameBound::Following(1),
+            ))];
+            let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0];
+            assert!(range.0.is_smallest());
+            assert_eq!(
+                range.1.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(2.into())])
+            );
+            assert_eq!(
+                range.2.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(5.into())])
+            );
+            assert_eq!(
+                range.3.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(5.into())])
+            );
+        }
+    }
+
+    #[test]
+    fn test_with_right_sentinel() {
+        let cache = create_cache!(1, 2, 4, 5, 8, ...);
+        let delta = create_delta!((3, Insert), (4, Insert), (5, Delete));
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Preceding(1),
+                FrameBound::Preceding(1),
+            ))];
+            assert_ranges_eq(
+                find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)),
+                [(2.into(), 3.into(), 8.into(), 8.into())],
+            );
+        }
+
+        {
+            let calls = vec![create_call(Frame::rows(
+                FrameBound::Preceding(1),
+                FrameBound::Following(1),
+            ))];
+            let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0];
+            assert_eq!(
+                range.0.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(1.into())])
+            );
+            assert_eq!(
+                range.1.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(2.into())])
+            );
+            assert_eq!(
+                range.2.as_normal_expect().pk.0,
+                OwnedRow::new(vec![Some(8.into())])
+            );
+            assert!(range.3.is_largest());
+        }
+    }
+}
diff --git a/src/stream/src/executor/over_window/sentinel.rs b/src/stream/src/executor/over_window/sentinel.rs
new file mode 100644
index 0000000000000..fd7ac2cf9749f
--- /dev/null
+++ b/src/stream/src/executor/over_window/sentinel.rs
@@ -0,0 +1,77 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use enum_as_inner::EnumAsInner;
+use risingwave_common::estimate_size::EstimateSize;
+use risingwave_expr::window_function::StateKey;
+
+#[derive(Debug, Clone, PartialEq, Eq, EnumAsInner)]
+pub(super) enum KeyWithSentinel<T> {
+    Smallest,
+    Normal(T),
+    Largest,
+}
+
+impl<T> KeyWithSentinel<T> {
+    pub fn as_normal_expect(&self) -> &T {
+        self.as_normal().expect("expect normal key")
+    }
+
+    pub fn is_sentinel(&self) -> bool {
+        matches!(self, Self::Smallest | Self::Largest)
+    }
+}
+
+impl<T> PartialOrd for KeyWithSentinel<T>
+where
+    T: Ord,
+{
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<T> Ord for KeyWithSentinel<T>
+where
+    T: Ord,
+{
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        use KeyWithSentinel::*;
+        match (self, other) {
+            (Smallest, Smallest) => std::cmp::Ordering::Equal,
+            (Smallest, _) => std::cmp::Ordering::Less,
+            (_, Smallest) => std::cmp::Ordering::Greater,
+            (Largest, Largest) => std::cmp::Ordering::Equal,
+            (Largest, _) => std::cmp::Ordering::Greater,
+            (_, Largest) => std::cmp::Ordering::Less,
+            (Normal(a), Normal(b)) => a.cmp(b),
+        }
+    }
+}
+
+impl<T: EstimateSize> EstimateSize for KeyWithSentinel<T> {
+    fn estimated_heap_size(&self) -> usize {
+        match self {
+            Self::Smallest => 0,
+            Self::Normal(inner) => inner.estimated_heap_size(),
+            Self::Largest => 0,
+        }
+    }
+}
+
+impl From<StateKey> for KeyWithSentinel<StateKey> {
+    fn from(key: StateKey) -> Self {
+        Self::Normal(key)
+    }
+}
diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs
index d9a1e9985c6bd..c13c9a86ca78e 100644
--- a/src/stream/src/executor/project.rs
+++ b/src/stream/src/executor/project.rs
@@ -47,7 +47,7 @@ struct Inner {
     /// Last seen values of nondecreasing expressions, buffered to periodically produce watermarks.
     last_nondec_expr_values: Vec<Option<ScalarImpl>>,
 
-    /// the selectivity threshold which should be in [0,1]. for the chunk with selectivity less
+    /// the selectivity threshold which should be in `[0,1]`. for the chunk with selectivity less
     /// than the threshold, the Project executor will construct a new chunk before expr evaluation,
     materialize_selectivity_threshold: f64,
 }
diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs
index df56643afb923..f1962d456b2e1 100644
--- a/src/stream/src/executor/project_set.rs
+++ b/src/stream/src/executor/project_set.rs
@@ -17,23 +17,56 @@ use std::fmt::{Debug, Formatter};
 use either::Either;
 use futures::StreamExt;
 use futures_async_stream::try_stream;
+use multimap::MultiMap;
 use risingwave_common::array::{Op, StreamChunk};
+use risingwave_common::bail;
 use risingwave_common::catalog::{Field, Schema};
-use risingwave_common::types::{DataType, DatumRef};
-use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+use risingwave_common::row::{Row, RowExt};
+use risingwave_common::types::{DataType, Datum, DatumRef, ToOwnedDatum};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_expr::table_function::ProjectSetSelectItem;
 
 use super::error::StreamExecutorError;
-use super::{BoxedExecutor, Executor, ExecutorInfo, Message, PkIndices, PkIndicesRef};
+use super::{
+    ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, PkIndices, PkIndicesRef,
+    StreamExecutorResult, Watermark,
+};
+use crate::common::StreamChunkBuilder;
+
+const PROJ_ROW_ID_OFFSET: usize = 1;
+
+/// `ProjectSetExecutor` projects data with the `expr`. The `expr` takes a chunk of data,
+/// and returns a new data chunk. And then, `ProjectSetExecutor` will insert, delete
+/// or update element into next operator according to the result of the expression.
+pub struct ProjectSetExecutor {
+    input: BoxedExecutor,
+    inner: Inner,
+}
+
+struct Inner {
+    info: ExecutorInfo,
+    ctx: ActorContextRef,
+    /// Expressions of the current project_section.
+    select_list: Vec<ProjectSetSelectItem>,
+    chunk_size: usize,
+    /// All the watermark derivations, (input_column_index, expr_idx). And the
+    /// derivation expression is the project_set's expression itself.
+    watermark_derivations: MultiMap<usize, usize>,
+    /// Indices of nondecreasing expressions in the expression list.
+    nondecreasing_expr_indices: Vec<usize>,
+}
 
 impl ProjectSetExecutor {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
+        ctx: ActorContextRef,
         input: Box<dyn Executor>,
         pk_indices: PkIndices,
         select_list: Vec<ProjectSetSelectItem>,
         executor_id: u64,
         chunk_size: usize,
+        watermark_derivations: MultiMap<usize, usize>,
+        nondecreasing_expr_indices: Vec<usize>,
     ) -> Self {
         let mut fields = vec![Field::with_name(DataType::Int64, "projected_row_id")];
         fields.extend(
@@ -47,76 +80,85 @@ impl ProjectSetExecutor {
             pk_indices,
             identity: format!("ProjectSet {:X}", executor_id),
         };
-        Self {
-            input,
+
+        let inner = Inner {
             info,
+            ctx,
             select_list,
             chunk_size,
-        }
-    }
-}
+            watermark_derivations,
+            nondecreasing_expr_indices,
+        };
 
-/// `ProjectSetExecutor` projects data with the `expr`. The `expr` takes a chunk of data,
-/// and returns a new data chunk. And then, `ProjectSetExecutor` will insert, delete
-/// or update element into next operator according to the result of the expression.
-pub struct ProjectSetExecutor {
-    input: BoxedExecutor,
-    info: ExecutorInfo,
-    /// Expressions of the current project_section.
-    select_list: Vec<ProjectSetSelectItem>,
-    chunk_size: usize,
+        Self { input, inner }
+    }
 }
 
 impl Debug for ProjectSetExecutor {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("ProjectSetExecutor")
-            .field("exprs", &self.select_list)
+            .field("exprs", &self.inner.select_list)
             .finish()
     }
 }
 
 impl Executor for ProjectSetExecutor {
     fn execute(self: Box<Self>) -> super::BoxedMessageStream {
-        self.execute_inner().boxed()
+        self.inner.execute(self.input).boxed()
     }
 
     fn schema(&self) -> &Schema {
-        &self.info.schema
+        &self.inner.info.schema
     }
 
     fn pk_indices(&self) -> PkIndicesRef<'_> {
-        &self.info.pk_indices
+        &self.inner.info.pk_indices
     }
 
     fn identity(&self) -> &str {
-        &self.info.identity
+        &self.inner.info.identity
     }
 }
 
-impl ProjectSetExecutor {
+impl Inner {
     #[try_stream(ok = Message, error = StreamExecutorError)]
-    async fn execute_inner(self) {
+    async fn execute(self, input: BoxedExecutor) {
         assert!(!self.select_list.is_empty());
-
         // First column will be `projected_row_id`, which represents the index in the
         // output table
-        let mut ops_builder = Vec::with_capacity(self.chunk_size);
-        let mut builder = DataChunkBuilder::new(
-            std::iter::once(DataType::Int64)
-                .chain(self.select_list.iter().map(|i| i.return_type()))
-                .collect(),
-            self.chunk_size,
-        );
+        let data_types: Vec<_> = std::iter::once(DataType::Int64)
+            .chain(self.select_list.iter().map(|i| i.return_type()))
+            .collect();
         // a temporary row buffer
-        let mut row = vec![None as DatumRef<'_>; builder.num_columns()];
+        let mut row = vec![DatumRef::None; data_types.len()];
+        let mut builder = StreamChunkBuilder::new(self.chunk_size, data_types);
 
+        let mut last_nondec_expr_values = vec![None; self.nondecreasing_expr_indices.len()];
         #[for_await]
-        for msg in self.input.execute() {
+        for msg in input.execute() {
             match msg? {
-                Message::Watermark(_) => {
-                    // TODO: https://github.com/risingwavelabs/risingwave/issues/6042
+                Message::Watermark(watermark) => {
+                    let watermarks = self.handle_watermark(watermark).await?;
+                    for watermark in watermarks {
+                        yield Message::Watermark(watermark)
+                    }
+                }
+                m @ Message::Barrier(_) => {
+                    for (&expr_idx, value) in self
+                        .nondecreasing_expr_indices
+                        .iter()
+                        .zip_eq_fast(&mut last_nondec_expr_values)
+                    {
+                        if let Some(value) = std::mem::take(value) {
+                            yield Message::Watermark(Watermark::new(
+                                expr_idx + PROJ_ROW_ID_OFFSET,
+                                self.select_list[expr_idx].return_type(),
+                                value,
+                            ))
+                        }
+                    }
+                    yield m
                 }
-                m @ Message::Barrier(_) => yield m,
                 Message::Chunk(chunk) => {
                     let mut results = Vec::with_capacity(self.select_list.len());
                     for select_item in &self.select_list {
@@ -159,13 +201,12 @@ impl ProjectSetExecutor {
                                 // no more output rows for the input row
                                 break;
                             }
-                            ops_builder.push(op);
-                            if let Some(chunk) = builder.append_one_row(&*row) {
-                                let ops = std::mem::replace(
-                                    &mut ops_builder,
-                                    Vec::with_capacity(self.chunk_size),
+                            if let Some(chunk) = builder.append_row(op, &*row) {
+                                self.update_last_nondec_expr_values(
+                                    &mut last_nondec_expr_values,
+                                    &chunk,
                                 );
-                                yield StreamChunk::from_parts(ops, chunk).into();
+                                yield Message::Chunk(chunk);
                             }
                             // move to the next row
                             for item in &mut results {
@@ -175,15 +216,72 @@ impl ProjectSetExecutor {
                             }
                         }
                     }
-                    if let Some(chunk) = builder.consume_all() {
-                        let ops = std::mem::replace(
-                            &mut ops_builder,
-                            Vec::with_capacity(self.chunk_size),
-                        );
-                        yield StreamChunk::from_parts(ops, chunk).into();
+                    if let Some(chunk) = builder.take() {
+                        self.update_last_nondec_expr_values(&mut last_nondec_expr_values, &chunk);
+                        yield Message::Chunk(chunk);
                     }
                 }
             }
         }
     }
+
+    fn update_last_nondec_expr_values(
+        &self,
+        last_nondec_expr_values: &mut [Datum],
+        chunk: &StreamChunk,
+    ) {
+        if !self.nondecreasing_expr_indices.is_empty() {
+            if let Some((_, first_visible_row)) = chunk.rows().next() {
+                // it's ok to use the first row here, just one chunk delay
+                first_visible_row
+                    .project(&self.nondecreasing_expr_indices)
+                    .iter()
+                    .enumerate()
+                    .for_each(|(idx, value)| {
+                        last_nondec_expr_values[idx] = Some(
+                            value
+                                .to_owned_datum()
+                                .expect("non-decreasing expression should never be NULL"),
+                        );
+                    });
+            }
+        }
+    }
+
+    async fn handle_watermark(&self, watermark: Watermark) -> StreamExecutorResult<Vec<Watermark>> {
+        let expr_indices = match self.watermark_derivations.get_vec(&watermark.col_idx) {
+            Some(v) => v,
+            None => return Ok(vec![]),
+        };
+        let mut ret = vec![];
+        for expr_idx in expr_indices {
+            let expr_idx = *expr_idx;
+            let derived_watermark = match &self.select_list[expr_idx] {
+                ProjectSetSelectItem::Expr(expr) => {
+                    watermark
+                        .clone()
+                        .transform_with_expr(expr, expr_idx + PROJ_ROW_ID_OFFSET, |err| {
+                            self.ctx.on_compute_error(
+                                err,
+                                &(self.info.identity.to_string() + "(when computing watermark)"),
+                            )
+                        })
+                        .await
+                }
+                ProjectSetSelectItem::TableFunction(_) => {
+                    bail!("Watermark should not be produced by a table function");
+                }
+            };
+
+            if let Some(derived_watermark) = derived_watermark {
+                ret.push(derived_watermark);
+            } else {
+                warn!(
+                    "{} derive a NULL watermark with the expression {}!",
+                    self.info.identity, expr_idx
+                );
+            }
+        }
+        Ok(ret)
+    }
 }
diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs
index e8364273a5375..1c0e7c6399a56 100644
--- a/src/stream/src/executor/receiver.rs
+++ b/src/stream/src/executor/receiver.rs
@@ -140,7 +140,7 @@ impl Executor for ReceiverExecutor {
                     }
                     Message::Barrier(barrier) => {
                         tracing::trace!(
-                            target: "events::barrier::path",
+                            target: "events::stream::barrier::path",
                             actor_id = actor_id,
                             "receiver receives barrier from path: {:?}",
                             barrier.passed_actors
diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs
index 19d7e2cb819ee..375c3587a8650 100644
--- a/src/stream/src/executor/sink.rs
+++ b/src/stream/src/executor/sink.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Instant;
 
@@ -23,11 +22,10 @@ use itertools::Itertools;
 use prometheus::Histogram;
 use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::catalog::{ColumnCatalog, Field, Schema};
-use risingwave_common::row::Row;
 use risingwave_common::types::DataType;
-use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+use risingwave_common::util::epoch::EpochPair;
 use risingwave_connector::dispatch_sink;
-use risingwave_connector::sink::catalog::{SinkId, SinkType};
+use risingwave_connector::sink::catalog::SinkType;
 use risingwave_connector::sink::{
     build_sink, Sink, SinkImpl, SinkParam, SinkWriter, SinkWriterParam,
 };
@@ -35,6 +33,7 @@ use risingwave_connector::sink::{
 use super::error::{StreamExecutorError, StreamExecutorResult};
 use super::{BoxedExecutor, Executor, Message};
 use crate::common::log_store::{LogReader, LogStoreFactory, LogStoreReadItem, LogWriter};
+use crate::common::StreamChunkBuilder;
 use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{expect_first_barrier, ActorContextRef, BoxedMessageStream};
 
@@ -58,17 +57,14 @@ struct SinkMetrics {
 
 // Drop all the DELETE messages in this chunk and convert UPDATE INSERT into INSERT.
 fn force_append_only(chunk: StreamChunk, data_types: Vec<DataType>) -> Option<StreamChunk> {
-    let mut builder = DataChunkBuilder::new(data_types, chunk.cardinality() + 1);
+    let mut builder = StreamChunkBuilder::new(chunk.cardinality() + 1, data_types);
     for (op, row_ref) in chunk.rows() {
         if op == Op::Insert || op == Op::UpdateInsert {
-            let finished = builder.append_one_row(row_ref.into_owned_row());
-            assert!(finished.is_none());
+            let none = builder.append_row(Op::Insert, row_ref);
+            assert!(none.is_none());
         }
     }
-    builder.consume_all().map(|data_chunk| {
-        let ops = vec![Op::Insert; data_chunk.capacity()];
-        StreamChunk::from_parts(ops, data_chunk)
-    })
+    builder.take()
 }
 
 impl<F: LogStoreFactory> SinkExecutor<F> {
@@ -77,27 +73,13 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
         input: BoxedExecutor,
         metrics: Arc<StreamingMetrics>,
         sink_writer_param: SinkWriterParam,
+        sink_param: SinkParam,
         columns: Vec<ColumnCatalog>,
-        properties: HashMap<String, String>,
-        pk_indices: Vec<usize>,
-        sink_type: SinkType,
-        sink_id: SinkId,
         actor_context: ActorContextRef,
         log_store_factory: F,
     ) -> StreamExecutorResult<Self> {
         let (log_reader, log_writer) = log_store_factory.build().await;
 
-        let sink_param = SinkParam {
-            sink_id,
-            properties,
-            columns: columns
-                .iter()
-                .filter(|col| !col.is_hidden)
-                .map(|col| col.column_desc.clone())
-                .collect(),
-            pk_indices,
-            sink_type,
-        };
         let sink = build_sink(sink_param.clone())?;
         let input_schema = columns
             .iter()
@@ -167,7 +149,9 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
 
         let epoch_pair = barrier.epoch;
 
-        log_writer.init(epoch_pair.curr).await?;
+        log_writer
+            .init(EpochPair::new_test_epoch(epoch_pair.curr))
+            .await?;
 
         // Propagate the first barrier
         yield Message::Barrier(barrier);
@@ -222,6 +206,7 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
             .filter_map(|(idx, column)| (!column.is_hidden).then_some(idx))
             .collect_vec();
 
+        #[derive(Debug)]
         enum LogConsumerState {
             /// Mark that the log consumer is not initialized yet
             Uninitialized,
@@ -237,6 +222,16 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
 
         loop {
             let (epoch, item): (u64, LogStoreReadItem) = log_reader.next_item().await?;
+            if let LogStoreReadItem::UpdateVnodeBitmap(_) = &item {
+                match &state {
+                    LogConsumerState::BarrierReceived { .. } => {}
+                    _ => unreachable!(
+                        "update vnode bitmap can be accepted only right after \
+                    barrier, but current state is {:?}",
+                        state
+                    ),
+                }
+            }
             // begin_epoch when not previously began
             state = match state {
                 LogConsumerState::Uninitialized => {
@@ -294,6 +289,9 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                     };
                     state = LogConsumerState::BarrierReceived { prev_epoch }
                 }
+                LogStoreReadItem::UpdateVnodeBitmap(vnode_bitmap) => {
+                    sink_writer.update_vnode_bitmap(vnode_bitmap).await?;
+                }
             }
         }
     }
@@ -385,15 +383,26 @@ mod test {
             ],
         );
 
+        let sink_param = SinkParam {
+            sink_id: 0.into(),
+            properties,
+            columns: columns
+                .iter()
+                .filter(|col| !col.is_hidden)
+                .map(|col| col.column_desc.clone())
+                .collect(),
+            pk_indices: pk.clone(),
+            sink_type: SinkType::ForceAppendOnly,
+            db_name: "test".into(),
+            sink_from_name: "test".into(),
+        };
+
         let sink_executor = SinkExecutor::new(
             Box::new(mock),
             Arc::new(StreamingMetrics::unused()),
             SinkWriterParam::default(),
+            sink_param,
             columns.clone(),
-            properties,
-            pk.clone(),
-            SinkType::ForceAppendOnly,
-            0.into(),
             ActorContext::create(0),
             BoundedInMemLogStoreFactory::new(1),
         )
@@ -471,15 +480,26 @@ mod test {
             ],
         );
 
+        let sink_param = SinkParam {
+            sink_id: 0.into(),
+            properties,
+            columns: columns
+                .iter()
+                .filter(|col| !col.is_hidden)
+                .map(|col| col.column_desc.clone())
+                .collect(),
+            pk_indices: pk.clone(),
+            sink_type: SinkType::ForceAppendOnly,
+            db_name: "test".into(),
+            sink_from_name: "test".into(),
+        };
+
         let sink_executor = SinkExecutor::new(
             Box::new(mock),
             Arc::new(StreamingMetrics::unused()),
             SinkWriterParam::default(),
+            sink_param,
             columns,
-            properties,
-            pk.clone(),
-            SinkType::ForceAppendOnly,
-            0.into(),
             ActorContext::create(0),
             BoundedInMemLogStoreFactory::new(1),
         )
diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs
index 4241ca74f89a3..1d2a2003d4e0a 100644
--- a/src/stream/src/executor/sort.rs
+++ b/src/stream/src/executor/sort.rs
@@ -18,13 +18,13 @@ use risingwave_common::array::Op;
 use risingwave_common::catalog::Schema;
 use risingwave_storage::StateStore;
 
-use super::aggregation::ChunkBuilder;
 use super::sort_buffer::SortBuffer;
 use super::{
     expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor,
     ExecutorInfo, Message, PkIndices, PkIndicesRef, StreamExecutorError, Watermark,
 };
 use crate::common::table::state_table::StateTable;
+use crate::common::StreamChunkBuilder;
 
 pub struct SortExecutor<S: StateStore> {
     input: BoxedExecutor,
@@ -122,7 +122,7 @@ impl<S: StateStore> SortExecutor<S> {
                     if col_idx == this.sort_column_index =>
                 {
                     let mut chunk_builder =
-                        ChunkBuilder::new(this.chunk_size, &this.info.schema.data_types());
+                        StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types());
 
                     #[for_await]
                     for row in vars
diff --git a/src/stream/src/executor/sort_buffer.rs b/src/stream/src/executor/sort_buffer.rs
index 2ecdc33e1e998..2a4cd4bf6ad78 100644
--- a/src/stream/src/executor/sort_buffer.rs
+++ b/src/stream/src/executor/sort_buffer.rs
@@ -28,9 +28,9 @@ use risingwave_common::types::{
     DefaultOrd, DefaultOrdered, ScalarImpl, ScalarRefImpl, ToOwnedDatum,
 };
 use risingwave_common::util::memcmp_encoding::MemcmpEncoded;
-use risingwave_storage::row_serde::row_serde_util::deserialize_pk_with_vnode;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::table::merge_sort::merge_sort;
+use risingwave_storage::table::KeyedRow;
 use risingwave_storage::StateStore;
 
 use super::{StreamExecutorError, StreamExecutorResult};
@@ -141,19 +141,19 @@ impl<S: StateStore> SortBuffer<S> {
         watermark: ScalarImpl,
         buffer_table: &'a mut StateTable<S>,
     ) {
-        let mut last_timestamp = None;
+        let mut last_table_pk = None;
         loop {
             if !self.cache.is_synced() {
                 // Refill the cache, then consume from the cache, to ensure strong row ordering
                 // and prefetch for the next watermark.
-                self.refill_cache(last_timestamp.take(), buffer_table)
+                self.refill_cache(last_table_pk.take(), buffer_table)
                     .await?;
             }
 
             #[for_await]
             for res in self.consume_from_cache(watermark.as_scalar_ref_impl()) {
-                let ((timestamp_val, _), row) = res?;
-                last_timestamp = Some(timestamp_val.into_inner());
+                let row = res?;
+                last_table_pk = Some((&row).project(buffer_table.pk_indices()).into_owned_row());
                 yield row;
             }
 
@@ -169,7 +169,7 @@ impl<S: StateStore> SortBuffer<S> {
         buffer_table.update_watermark(watermark, true);
     }
 
-    #[try_stream(ok = (CacheKey, OwnedRow), error = StreamExecutorError)]
+    #[try_stream(ok = OwnedRow, error = StreamExecutorError)]
     async fn consume_from_cache<'a>(&'a mut self, watermark: ScalarRefImpl<'a>) {
         while self.cache.is_synced() {
             let Some(key) = self.cache.first_key_value().map(|(k, _)| k.clone()) else {
@@ -177,7 +177,7 @@ impl<S: StateStore> SortBuffer<S> {
             };
             if key.0.as_scalar_ref_impl().default_cmp(&watermark).is_lt() {
                 let row = self.cache.delete(&key).unwrap();
-                yield (key, row);
+                yield row;
             } else {
                 break;
             }
@@ -187,27 +187,24 @@ impl<S: StateStore> SortBuffer<S> {
     /// Clear the cache and refill it with the current content of the buffer table.
     pub async fn refill_cache(
         &mut self,
-        last_timestamp: Option<ScalarImpl>,
+        last_table_pk: Option<OwnedRow>,
         buffer_table: &StateTable<S>,
     ) -> StreamExecutorResult<()> {
         let mut filler = self.cache.begin_syncing();
 
         let pk_range = (
-            last_timestamp
-                .as_ref()
-                .map(|v| Bound::Excluded([Some(v.as_scalar_ref_impl())]))
+            last_table_pk
+                .map(Bound::Excluded)
                 .unwrap_or(Bound::Unbounded),
             Bound::<row::Empty>::Unbounded,
         );
 
         let streams: Vec<_> =
             futures::future::try_join_all(buffer_table.vnode_bitmap().iter_vnodes().map(|vnode| {
-                buffer_table.iter_key_and_val_with_pk_range(
+                buffer_table.iter_row_with_pk_range(
                     &pk_range,
                     vnode,
-                    PrefetchOptions {
-                        exhaust_iter: filler.capacity().is_none(),
-                    },
+                    PrefetchOptions::new_with_exhaust_iter(filler.capacity().is_none()),
                 )
             }))
             .await?
@@ -230,11 +227,11 @@ impl<S: StateStore> SortBuffer<S> {
 /// Merge the key part and value part of a row into a full row. This is needed for state table with
 /// non-None value indices.
 fn key_value_to_full_row<S: StateStore>(
-    (key, value): (Bytes, OwnedRow),
+    keyed_row: KeyedRow<Bytes>,
     table: &StateTable<S>,
 ) -> StreamExecutorResult<OwnedRow> {
     let Some(val_indices) = table.value_indices() else {
-        return Ok(value);
+        return Ok(keyed_row.into_owned_row());
     };
     let pk_indices = table.pk_indices();
     let indices: BTreeSet<_> = val_indices
@@ -246,13 +243,14 @@ fn key_value_to_full_row<S: StateStore>(
     assert!(indices.iter().copied().eq(0..len));
 
     let mut row = vec![None; len];
-    let key = deserialize_pk_with_vnode(&key, table.pk_serde())
-        .map_err(|e| anyhow!("failed to deserialize pk: {}", e))?
-        .1;
+    let key = table
+        .pk_serde()
+        .deserialize(keyed_row.key())
+        .map_err(|e| anyhow!("failed to deserialize pk: {}", e))?;
     for (i, v) in key.into_iter().enumerate() {
         row[pk_indices[i]] = v;
     }
-    for (i, v) in value.into_iter().enumerate() {
+    for (i, v) in keyed_row.into_owned_row().into_iter().enumerate() {
         row[val_indices[i]] = v;
     }
     Ok(OwnedRow::new(row))
diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs
index dd70e908eff03..aba89f817b09a 100644
--- a/src/stream/src/executor/source/fs_source_executor.rs
+++ b/src/stream/src/executor/source/fs_source_executor.rs
@@ -279,9 +279,8 @@ impl<S: StateStore> FsSourceExecutor<S> {
             .build_fs_source_desc()
             .map_err(StreamExecutorError::connector_error)?;
 
-        // If the first barrier is configuration change, then the source executor must be newly
-        // created, and we should start with the paused state.
-        let start_with_paused = barrier.is_update();
+        // If the first barrier requires us to pause on startup, pause the stream.
+        let start_with_paused = barrier.is_pause_on_startup();
 
         let mut boot_state = Vec::default();
         if let Some(mutation) = barrier.mutation.as_deref() {
diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs
index 33182fca3e8fd..44243334e1d18 100644
--- a/src/stream/src/executor/source/source_executor.rs
+++ b/src/stream/src/executor/source/source_executor.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::fmt::Formatter;
+use std::time::Duration;
 
 use anyhow::anyhow;
 use either::Either;
@@ -120,40 +121,6 @@ impl<S: StateStore> SourceExecutor<S> {
             .map_err(StreamExecutorError::connector_error)
     }
 
-    fn check_split_assignment_is_migration(
-        &self,
-        actor_splits: &HashMap<ActorId, Vec<SplitImpl>>,
-    ) -> bool {
-        let core = self.stream_source_core.as_ref().unwrap();
-
-        let mut split_to_actors_index = HashMap::new();
-
-        for (actor_id, splits) in actor_splits {
-            for split in splits {
-                split_to_actors_index
-                    .entry(split.id())
-                    .or_insert(vec![])
-                    .push(*actor_id);
-            }
-        }
-
-        for split_id in core.state_cache.keys() {
-            if let Some(actor_ids) = split_to_actors_index.remove(split_id) {
-                if !actor_ids.contains(&self.actor_ctx.id) {
-                    tracing::warn!(
-                        "split {} migration from {} detected, target might be {:?}",
-                        split_id,
-                        self.actor_ctx.id,
-                        actor_ids
-                    );
-                    return true;
-                }
-            }
-        }
-
-        false
-    }
-
     #[inline]
     fn get_metric_labels(&self) -> [String; 3] {
         [
@@ -263,6 +230,54 @@ impl<S: StateStore> SourceExecutor<S> {
         Ok(split_changed.then_some(target_state))
     }
 
+    /// Rebuild stream if there is a err in stream
+    async fn rebuild_stream_reader_from_error<const BIASED: bool>(
+        &mut self,
+        source_desc: &SourceDesc,
+        stream: &mut StreamReaderWithPause<BIASED, StreamChunkWithState>,
+        split_info: &mut [SplitImpl],
+        e: StreamExecutorError,
+    ) -> StreamExecutorResult<()> {
+        let core = self.stream_source_core.as_mut().unwrap();
+        tracing::warn!(
+            "stream source reader error, actor: {:?}, source: {:?}",
+            self.actor_ctx.id,
+            core.source_id,
+        );
+        self.metrics
+            .user_source_reader_error_count
+            .with_label_values(&[
+                "SourceReaderError",
+                &e.to_string(),
+                "SourceExecutor",
+                &self.actor_ctx.id.to_string(),
+                &core.source_id.to_string(),
+            ])
+            .inc_by(1);
+        // fetch the newest offset, either it's in cache (before barrier)
+        // or in state table (just after barrier)
+        let target_state = if core.state_cache.is_empty() {
+            for ele in split_info.iter_mut() {
+                if let Some(recover_state) = core
+                    .split_state_store
+                    .try_recover_from_state_store(ele)
+                    .await?
+                {
+                    *ele = recover_state;
+                }
+            }
+            split_info.to_owned()
+        } else {
+            core.state_cache
+                .values()
+                .map(|split_impl| split_impl.to_owned())
+                .collect_vec()
+        };
+
+        self.replace_stream_reader_with_target_state(source_desc, stream, target_state)
+            .await
+    }
+
     async fn replace_stream_reader_with_target_state<const BIASED: bool>(
         &mut self,
         source_desc: &SourceDesc,
@@ -370,12 +385,18 @@ impl<S: StateStore> SourceExecutor<S> {
                     ..
                 } => {
                     if let Some(splits) = splits.get(&self.actor_ctx.id) {
+                        tracing::info!(
+                            "source exector: actor {:?} boot with splits: {:?}",
+                            self.actor_ctx.id,
+                            splits
+                        );
                         boot_state = splits.clone();
                     }
                 }
                 _ => {}
             }
         }
+        let mut latest_split_info = boot_state.clone();
 
         core.split_state_store.init_epoch(barrier.epoch);
 
@@ -410,9 +431,8 @@ impl<S: StateStore> SourceExecutor<S> {
             source_chunk_reader,
         );
 
-        // If the first barrier is configuration change, then the source executor must be newly
-        // created, and we should start with the paused state.
-        if barrier.is_update() {
+        // If the first barrier requires us to pause on startup, pause the stream.
+        if barrier.is_pause_on_startup() {
             stream.pause_stream();
         }
 
@@ -425,140 +445,171 @@ impl<S: StateStore> SourceExecutor<S> {
         let mut last_barrier_time = Instant::now();
         let mut self_paused = false;
         let mut metric_row_per_barrier: u64 = 0;
+
         while let Some(msg) = stream.next().await {
-            match msg? {
-                // This branch will be preferred.
-                Either::Left(msg) => match &msg {
-                    Message::Barrier(barrier) => {
-                        last_barrier_time = Instant::now();
-
-                        if self_paused {
-                            stream.resume_stream();
-                            self_paused = false;
-                        }
+            match msg {
+                Err(e) => {
+                    tokio::time::sleep(Duration::from_millis(1000)).await;
+                    self.rebuild_stream_reader_from_error(
+                        &source_desc,
+                        &mut stream,
+                        &mut latest_split_info,
+                        e,
+                    )
+                    .await?;
+                }
+                Ok(msg) => {
+                    match msg {
+                        // This branch will be preferred.
+                        Either::Left(msg) => match &msg {
+                            Message::Barrier(barrier) => {
+                                last_barrier_time = Instant::now();
+
+                                if self_paused {
+                                    stream.resume_stream();
+                                    self_paused = false;
+                                }
 
-                        let epoch = barrier.epoch;
-
-                        let mut target_state = None;
-                        let mut should_trim_state = false;
-
-                        if let Some(ref mutation) = barrier.mutation.as_deref() {
-                            match mutation {
-                                Mutation::Pause => stream.pause_stream(),
-                                Mutation::Resume => stream.resume_stream(),
-                                Mutation::SourceChangeSplit(actor_splits) => {
-                                    tracing::info!(
-                                        actor_id = self.actor_ctx.id,
-                                        actor_splits = ?actor_splits,
-                                        "source change split received"
-                                    );
-
-                                    // In the context of split changes, we do not allow split
-                                    // migration because it can lead to inconsistent states.
-                                    // Therefore, all split migration must be done via update
-                                    // mutation and pause/resume
-                                    assert!(!self.check_split_assignment_is_migration(actor_splits));
-
-                                    target_state = self
-                                        .apply_split_change(&source_desc, &mut stream, actor_splits)
-                                        .await?;
-                                    should_trim_state = true;
+                                let epoch = barrier.epoch;
+
+                                let mut target_state = None;
+                                let mut should_trim_state = false;
+
+                                if let Some(ref mutation) = barrier.mutation.as_deref() {
+                                    match mutation {
+                                        Mutation::Pause => stream.pause_stream(),
+                                        Mutation::Resume => stream.resume_stream(),
+                                        Mutation::SourceChangeSplit(actor_splits) => {
+                                            tracing::info!(
+                                                actor_id = self.actor_ctx.id,
+                                                actor_splits = ?actor_splits,
+                                                "source change split received"
+                                            );
+
+                                            target_state = self
+                                                .apply_split_change(
+                                                    &source_desc,
+                                                    &mut stream,
+                                                    actor_splits,
+                                                )
+                                                .await?;
+                                            should_trim_state = true;
+                                        }
+
+                                        Mutation::Update { actor_splits, .. } => {
+                                            target_state = self
+                                                .apply_split_change(
+                                                    &source_desc,
+                                                    &mut stream,
+                                                    actor_splits,
+                                                )
+                                                .await?;
+                                        }
+                                        _ => {}
+                                    }
                                 }
 
-                                Mutation::Update { actor_splits, .. } => {
-                                    target_state = self
-                                        .apply_split_change(&source_desc, &mut stream, actor_splits)
-                                        .await?;
+                                if let Some(target_state) = &target_state {
+                                    latest_split_info = target_state.clone();
                                 }
-                                _ => {}
-                            }
-                        }
 
-                        self.take_snapshot_and_clear_cache(epoch, target_state, should_trim_state)
-                            .await?;
+                                self.take_snapshot_and_clear_cache(
+                                    epoch,
+                                    target_state,
+                                    should_trim_state,
+                                )
+                                .await?;
+
+                                self.metrics
+                                    .source_row_per_barrier
+                                    .with_label_values(&[
+                                        self.actor_ctx.id.to_string().as_str(),
+                                        self.stream_source_core
+                                            .as_ref()
+                                            .unwrap()
+                                            .source_id
+                                            .to_string()
+                                            .as_ref(),
+                                    ])
+                                    .inc_by(metric_row_per_barrier);
+                                metric_row_per_barrier = 0;
+
+                                yield msg;
+                            }
+                            _ => {
+                                // For the source executor, the message we receive from this arm
+                                // should always be barrier message.
+                                unreachable!();
+                            }
+                        },
+
+                        Either::Right(StreamChunkWithState {
+                            chunk,
+                            split_offset_mapping,
+                        }) => {
+                            if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms {
+                                // Exceeds the max wait barrier time, the source will be paused.
+                                // Currently we can guarantee the
+                                // source is not paused since it received stream
+                                // chunks.
+                                self_paused = true;
+                                tracing::warn!(
+                                    "source {} paused, wait barrier for {:?}",
+                                    self.identity,
+                                    last_barrier_time.elapsed()
+                                );
+                                stream.pause_stream();
+
+                                // Only update `max_wait_barrier_time_ms` to capture
+                                // `barrier_interval_ms`
+                                // changes here to avoid frequently accessing the shared
+                                // `system_params`.
+                                max_wait_barrier_time_ms =
+                                    self.system_params.load().barrier_interval_ms() as u128
+                                        * WAIT_BARRIER_MULTIPLE_TIMES;
+                            }
+                            if let Some(mapping) = split_offset_mapping {
+                                let state: HashMap<_, _> = mapping
+                                    .iter()
+                                    .flat_map(|(split_id, offset)| {
+                                        let origin_split_impl = self
+                                            .stream_source_core
+                                            .as_mut()
+                                            .unwrap()
+                                            .stream_source_splits
+                                            .get_mut(split_id);
+
+                                        origin_split_impl.map(|split_impl| {
+                                            split_impl.update_in_place(offset.clone())?;
+                                            Ok::<_, anyhow::Error>((
+                                                split_id.clone(),
+                                                split_impl.clone(),
+                                            ))
+                                        })
+                                    })
+                                    .try_collect()?;
 
-                        self.metrics
-                            .source_row_per_barrier
-                            .with_label_values(&[
-                                self.actor_ctx.id.to_string().as_str(),
                                 self.stream_source_core
-                                    .as_ref()
-                                    .unwrap()
-                                    .source_id
-                                    .to_string()
-                                    .as_ref(),
-                            ])
-                            .inc_by(metric_row_per_barrier);
-                        metric_row_per_barrier = 0;
-
-                        yield msg;
-                    }
-                    _ => {
-                        // For the source executor, the message we receive from this arm should
-                        // always be barrier message.
-                        unreachable!();
-                    }
-                },
-
-                Either::Right(StreamChunkWithState {
-                    chunk,
-                    split_offset_mapping,
-                }) => {
-                    if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms {
-                        // Exceeds the max wait barrier time, the source will be paused. Currently
-                        // we can guarantee the source is not paused since it received stream
-                        // chunks.
-                        self_paused = true;
-                        tracing::warn!(
-                            "source {} paused, wait barrier for {:?}",
-                            self.identity,
-                            last_barrier_time.elapsed()
-                        );
-                        stream.pause_stream();
-
-                        // Only update `max_wait_barrier_time_ms` to capture `barrier_interval_ms`
-                        // changes here to avoid frequently accessing the shared `system_params`.
-                        max_wait_barrier_time_ms = self.system_params.load().barrier_interval_ms()
-                            as u128
-                            * WAIT_BARRIER_MULTIPLE_TIMES;
-                    }
-                    if let Some(mapping) = split_offset_mapping {
-                        let state: HashMap<_, _> = mapping
-                            .iter()
-                            .flat_map(|(split_id, offset)| {
-                                let origin_split_impl = self
-                                    .stream_source_core
                                     .as_mut()
                                     .unwrap()
-                                    .stream_source_splits
-                                    .get_mut(split_id);
-
-                                origin_split_impl.map(|split_impl| {
-                                    split_impl.update_in_place(offset.clone())?;
-                                    Ok::<_, anyhow::Error>((split_id.clone(), split_impl.clone()))
-                                })
-                            })
-                            .try_collect()?;
-
-                        self.stream_source_core
-                            .as_mut()
-                            .unwrap()
-                            .state_cache
-                            .extend(state);
+                                    .state_cache
+                                    .extend(state);
+                            }
+                            metric_row_per_barrier += chunk.cardinality() as u64;
+
+                            self.metrics
+                                .source_output_row_count
+                                .with_label_values(
+                                    &self
+                                        .get_metric_labels()
+                                        .iter()
+                                        .map(AsRef::as_ref)
+                                        .collect::<Vec<&str>>(),
+                                )
+                                .inc_by(chunk.cardinality() as u64);
+                            yield Message::Chunk(chunk);
+                        }
                     }
-                    metric_row_per_barrier += chunk.cardinality() as u64;
-
-                    self.metrics
-                        .source_output_row_count
-                        .with_label_values(
-                            &self
-                                .get_metric_labels()
-                                .iter()
-                                .map(AsRef::as_ref)
-                                .collect::<Vec<&str>>(),
-                        )
-                        .inc_by(chunk.cardinality() as u64);
-                    yield Message::Chunk(chunk);
                 }
             }
         }
@@ -721,6 +772,7 @@ mod tests {
                     }),
                 ],
             },
+            pause: false,
         });
         barrier_tx.send(init_barrier).unwrap();
 
@@ -812,6 +864,7 @@ mod tests {
                     }),
                 ],
             },
+            pause: false,
         });
         barrier_tx.send(init_barrier).unwrap();
 
diff --git a/src/stream/src/executor/source/state_table_handler.rs b/src/stream/src/executor/source/state_table_handler.rs
index 1002e6610e09b..f1ee9f0c90d4b 100644
--- a/src/stream/src/executor/source/state_table_handler.rs
+++ b/src/stream/src/executor/source/state_table_handler.rs
@@ -84,7 +84,7 @@ impl<S: StateStore> SourceStateTableHandler<S> {
         // all source executor has vnode id zero
         let iter = self
             .state_store
-            .iter_with_pk_range(
+            .iter_row_with_pk_range(
                 &(start, end),
                 VirtualNode::ZERO,
                 PrefetchOptions::new_for_exhaust_iter(),
@@ -93,8 +93,8 @@ impl<S: StateStore> SourceStateTableHandler<S> {
 
         let mut set = HashSet::new();
         pin_mut!(iter);
-        while let Some(row) = iter.next().await {
-            let row = row?;
+        while let Some(keyed_row) = iter.next().await {
+            let row = keyed_row?;
             if let Some(ScalarRefImpl::Jsonb(jsonb_ref)) = row.datum_at(1) {
                 let split = SplitImpl::restore_from_json(jsonb_ref.to_owned_scalar())?;
                 let fs = split
@@ -143,7 +143,7 @@ impl<S: StateStore> SourceStateTableHandler<S> {
         Ok(())
     }
 
-    async fn set(&mut self, key: SplitId, value: JsonbVal) -> StreamExecutorResult<()> {
+    pub async fn set(&mut self, key: SplitId, value: JsonbVal) -> StreamExecutorResult<()> {
         let row = [
             Some(Self::string_to_scalar(key.deref())),
             Some(ScalarImpl::Jsonb(value)),
diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs
index cdb866495e11e..6f82ca6e75adf 100644
--- a/src/stream/src/executor/temporal_join.rs
+++ b/src/stream/src/executor/temporal_join.rs
@@ -42,7 +42,7 @@ use risingwave_storage::StateStore;
 use super::{Barrier, Executor, Message, MessageStream, StreamExecutorError, StreamExecutorResult};
 use crate::cache::{cache_may_stale, new_with_hasher_in, ManagedLruCache};
 use crate::common::metrics::MetricsInfo;
-use crate::common::StreamChunkBuilder;
+use crate::common::JoinStreamChunkBuilder;
 use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{ActorContextRef, BoxedExecutor, JoinType, JoinTypePrimitive, PkIndices};
 use crate::task::AtomicU64Ref;
@@ -375,8 +375,8 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> TemporalJoinExecutor
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn into_stream(mut self) {
-        let (left_map, right_map) = StreamChunkBuilder::get_i2o_mapping(
-            self.output_indices.iter().cloned(),
+        let (left_map, right_map) = JoinStreamChunkBuilder::get_i2o_mapping(
+            &self.output_indices,
             self.left.schema().len(),
             self.right.schema().len(),
         );
@@ -401,9 +401,9 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> TemporalJoinExecutor
                 InternalMessage::Chunk(chunk) => {
                     // Compact chunk, otherwise the following keys and chunk rows might fail to zip.
                     let chunk = chunk.compact();
-                    let mut builder = StreamChunkBuilder::new(
+                    let mut builder = JoinStreamChunkBuilder::new(
                         self.chunk_size,
-                        &self.schema.data_types(),
+                        self.schema.data_types(),
                         left_map.clone(),
                         right_map.clone(),
                     );
diff --git a/src/stream/src/executor/top_n/mod.rs b/src/stream/src/executor/top_n/mod.rs
index 41d5a905a2960..ad743f564d2d6 100644
--- a/src/stream/src/executor/top_n/mod.rs
+++ b/src/stream/src/executor/top_n/mod.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// Wrapper and helper functions to help implement [`Executor`] for `TopN` variants
+/// Wrapper and helper functions to help implement [`crate::executor::Executor`] for `TopN` variants
 mod utils;
 use utils::*;
 
diff --git a/src/stream/src/executor/top_n/top_n_state.rs b/src/stream/src/executor/top_n/top_n_state.rs
index 53f15efa85363..87d19e8550861 100644
--- a/src/stream/src/executor/top_n/top_n_state.rs
+++ b/src/stream/src/executor/top_n/top_n_state.rs
@@ -83,7 +83,7 @@ impl<S: StateStore> ManagedTopNState<S> {
     ) -> StreamExecutorResult<Vec<TopNStateRow>> {
         let state_table_iter = self
             .state_table
-            .iter_with_pk_prefix(&group_key, Default::default())
+            .iter_row_with_pk_prefix(&group_key, Default::default())
             .await?;
         pin_mut!(state_table_iter);
 
@@ -100,7 +100,7 @@ impl<S: StateStore> ManagedTopNState<S> {
             )
         };
         while let Some(item) = stream.next().await {
-            rows.push(self.get_topn_row(item?, group_key.len()));
+            rows.push(self.get_topn_row(item?.into_owned_row(), group_key.len()));
         }
         Ok(rows)
     }
@@ -120,7 +120,7 @@ impl<S: StateStore> ManagedTopNState<S> {
         let cache = &mut topn_cache.high;
         let state_table_iter = self
             .state_table
-            .iter_with_pk_prefix(
+            .iter_row_with_pk_prefix(
                 &group_key,
                 PrefetchOptions {
                     exhaust_iter: cache_size_limit == usize::MAX,
@@ -130,7 +130,7 @@ impl<S: StateStore> ManagedTopNState<S> {
         pin_mut!(state_table_iter);
         while let Some(item) = state_table_iter.next().await {
             // Note(bugen): should first compare with start key before constructing TopNStateRow.
-            let topn_row = self.get_topn_row(item?, group_key.len());
+            let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
             if let Some(start_key) = start_key.as_ref() && &topn_row.cache_key <= start_key {
                 continue;
             }
@@ -143,7 +143,7 @@ impl<S: StateStore> ManagedTopNState<S> {
         if WITH_TIES && topn_cache.is_high_cache_full() {
             let high_last_sort_key = topn_cache.high.last_key_value().unwrap().0 .0.clone();
             while let Some(item) = state_table_iter.next().await {
-                let topn_row = self.get_topn_row(item?, group_key.len());
+                let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
                 if topn_row.cache_key.0 == high_last_sort_key {
                     topn_cache
                         .high
@@ -168,7 +168,7 @@ impl<S: StateStore> ManagedTopNState<S> {
 
         let state_table_iter = self
             .state_table
-            .iter_with_pk_prefix(
+            .iter_row_with_pk_prefix(
                 &group_key,
                 PrefetchOptions {
                     exhaust_iter: topn_cache.limit == usize::MAX,
@@ -178,7 +178,7 @@ impl<S: StateStore> ManagedTopNState<S> {
         pin_mut!(state_table_iter);
         if topn_cache.offset > 0 {
             while let Some(item) = state_table_iter.next().await {
-                let topn_row = self.get_topn_row(item?, group_key.len());
+                let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
                 topn_cache
                     .low
                     .insert(topn_row.cache_key, (&topn_row.row).into());
@@ -190,7 +190,7 @@ impl<S: StateStore> ManagedTopNState<S> {
 
         assert!(topn_cache.limit > 0, "topn cache limit should always > 0");
         while let Some(item) = state_table_iter.next().await {
-            let topn_row = self.get_topn_row(item?, group_key.len());
+            let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
             topn_cache
                 .middle
                 .insert(topn_row.cache_key, (&topn_row.row).into());
@@ -201,7 +201,7 @@ impl<S: StateStore> ManagedTopNState<S> {
         if WITH_TIES && topn_cache.is_middle_cache_full() {
             let middle_last_sort_key = topn_cache.middle.last_key_value().unwrap().0 .0.clone();
             while let Some(item) = state_table_iter.next().await {
-                let topn_row = self.get_topn_row(item?, group_key.len());
+                let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
                 if topn_row.cache_key.0 == middle_last_sort_key {
                     topn_cache
                         .middle
@@ -220,7 +220,7 @@ impl<S: StateStore> ManagedTopNState<S> {
             "topn cache high_capacity should always > 0"
         );
         while !topn_cache.is_high_cache_full() && let Some(item) = state_table_iter.next().await {
-            let topn_row = self.get_topn_row(item?, group_key.len());
+            let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
             topn_cache
                 .high
                 .insert(topn_row.cache_key, (&topn_row.row).into());
@@ -228,7 +228,7 @@ impl<S: StateStore> ManagedTopNState<S> {
         if WITH_TIES && topn_cache.is_high_cache_full() {
             let high_last_sort_key = topn_cache.high.last_key_value().unwrap().0 .0.clone();
             while let Some(item) = state_table_iter.next().await {
-                let topn_row = self.get_topn_row(item?, group_key.len());
+                let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
                 if topn_row.cache_key.0 == high_last_sort_key {
                     topn_cache
                         .high
diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs
index 29a9f8a77bee3..512e9f6c28da3 100644
--- a/src/stream/src/executor/values.rs
+++ b/src/stream/src/executor/values.rs
@@ -218,6 +218,7 @@ mod tests {
             adds: Default::default(),
             added_actors: maplit::hashset! {actor_id},
             splits: Default::default(),
+            pause: false,
         });
         tx.send(first_message).unwrap();
 
diff --git a/src/stream/src/executor/wrapper/trace.rs b/src/stream/src/executor/wrapper/trace.rs
index dc936b61c7e1c..0b18d54a0bb58 100644
--- a/src/stream/src/executor/wrapper/trace.rs
+++ b/src/stream/src/executor/wrapper/trace.rs
@@ -47,20 +47,45 @@ pub async fn trace(
     pin_mut!(input);
 
     while let Some(message) = input.next().instrument(span.clone()).await.transpose()? {
-        if let Message::Chunk(chunk) = &message {
-            if chunk.cardinality() > 0 && (enable_executor_row_count || is_sink_or_mv) {
-                metrics
-                    .executor_row_count
-                    .with_label_values(&[&actor_id_string, &span_name])
-                    .inc_by(chunk.cardinality() as u64);
-                tracing::trace!(?chunk, "chunk");
+        // Trace the message in the span's scope.
+        span.in_scope(|| match &message {
+            Message::Chunk(chunk) => {
+                if chunk.cardinality() > 0 {
+                    if enable_executor_row_count || is_sink_or_mv {
+                        metrics
+                            .executor_row_count
+                            .with_label_values(&[&actor_id_string, &span_name])
+                            .inc_by(chunk.cardinality() as u64);
+                    }
+                    tracing::trace!(
+                        target: "events::stream::message::chunk",
+                        cardinality = chunk.cardinality(),
+                        capacity = chunk.capacity(),
+                        "\n{}\n", chunk.to_pretty_with_schema(&info.schema),
+                    );
+                }
             }
-        }
+            Message::Watermark(watermark) => {
+                tracing::trace!(
+                    target: "events::stream::message::watermark",
+                    value = ?watermark.val,
+                    col_idx = watermark.col_idx,
+                );
+            }
+            Message::Barrier(barrier) => {
+                tracing::trace!(
+                    target: "events::stream::message::barrier",
+                    prev_epoch = barrier.epoch.prev,
+                    curr_epoch = barrier.epoch.curr,
+                    kind = ?barrier.kind,
+                );
+            }
+        });
 
+        // Yield the message and update the span.
         match &message {
             Message::Chunk(_) | Message::Watermark(_) => yield message,
-
-            Message::Barrier(_barrier) => {
+            Message::Barrier(_) => {
                 // Drop the span as the inner executor has finished processing the barrier (then all
                 // data from the previous epoch).
                 let _ = std::mem::replace(&mut span, Span::none());
diff --git a/src/stream/src/from_proto/chain.rs b/src/stream/src/from_proto/chain.rs
index 858eeb8954912..d1a971a5cbb4a 100644
--- a/src/stream/src/from_proto/chain.rs
+++ b/src/stream/src/from_proto/chain.rs
@@ -23,7 +23,9 @@ use risingwave_storage::table::Distribution;
 
 use super::*;
 use crate::common::table::state_table::StateTable;
-use crate::executor::{BackfillExecutor, ChainExecutor, RearrangedChainExecutor};
+use crate::executor::{
+    BackfillExecutor, ChainExecutor, FlowControlExecutor, RearrangedChainExecutor,
+};
 
 pub struct ChainExecutorBuilder;
 
@@ -172,6 +174,10 @@ impl ExecutorBuilder for ChainExecutorBuilder {
             }
             ChainType::ChainUnspecified => unreachable!(),
         };
-        Ok(executor)
+        if let Ok(rate_limit) = node.get_rate_limit() {
+            Ok(FlowControlExecutor::new(executor, *rate_limit).boxed())
+        } else {
+            Ok(executor)
+        }
     }
 }
diff --git a/src/stream/src/from_proto/over_window.rs b/src/stream/src/from_proto/over_window.rs
index b5459c4b99756..7d139ca3f74db 100644
--- a/src/stream/src/from_proto/over_window.rs
+++ b/src/stream/src/from_proto/over_window.rs
@@ -14,6 +14,7 @@
 
 use std::sync::Arc;
 
+use risingwave_common::session_config::OverWindowCachePolicy;
 use risingwave_common::util::sort_util::ColumnOrder;
 use risingwave_expr::function::window::WindowFuncCall;
 use risingwave_pb::stream_plan::PbOverWindowNode;
@@ -72,7 +73,11 @@ impl ExecutorBuilder for OverWindowExecutorBuilder {
             order_key_order_types,
             state_table,
             watermark_epoch: stream.get_watermark_epoch(),
+            metrics: params.executor_stats,
             chunk_size: params.env.config().developer.chunk_size,
+            cache_policy: OverWindowCachePolicy::from_protobuf(
+                node.get_cache_policy().unwrap_or_default(),
+            ),
         })
         .boxed())
     }
diff --git a/src/stream/src/from_proto/project_set.rs b/src/stream/src/from_proto/project_set.rs
index 4a07a3f8bed27..57c422169e54f 100644
--- a/src/stream/src/from_proto/project_set.rs
+++ b/src/stream/src/from_proto/project_set.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use multimap::MultiMap;
+use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_expr::table_function::ProjectSetSelectItem;
 use risingwave_pb::stream_plan::ProjectSetNode;
 
@@ -38,13 +40,32 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder {
                 ProjectSetSelectItem::from_prost(proto, params.env.config().developer.chunk_size)
             })
             .try_collect()?;
+        let watermark_derivations = MultiMap::from_iter(
+            node.get_watermark_input_cols()
+                .iter()
+                .map(|idx| *idx as usize)
+                .zip_eq_fast(
+                    node.get_watermark_expr_indices()
+                        .iter()
+                        .map(|idx| *idx as usize),
+                ),
+        );
+        let nondecreasing_expr_indices = node
+            .get_nondecreasing_exprs()
+            .iter()
+            .map(|idx| *idx as usize)
+            .collect();
+
         let chunk_size = params.env.config().developer.chunk_size;
         Ok(ProjectSetExecutor::new(
+            params.actor_context,
             input,
             params.pk_indices,
             select_list,
             params.executor_id,
             chunk_size,
+            watermark_derivations,
+            nondecreasing_expr_indices,
         )
         .boxed())
     }
diff --git a/src/stream/src/from_proto/sink.rs b/src/stream/src/from_proto/sink.rs
index d3f133822769f..0dd05d722b131 100644
--- a/src/stream/src/from_proto/sink.rs
+++ b/src/stream/src/from_proto/sink.rs
@@ -16,7 +16,7 @@ use std::sync::Arc;
 
 use risingwave_common::catalog::ColumnCatalog;
 use risingwave_connector::sink::catalog::SinkType;
-use risingwave_connector::sink::SinkWriterParam;
+use risingwave_connector::sink::{SinkParam, SinkWriterParam};
 use risingwave_pb::stream_plan::{SinkLogStoreType, SinkNode};
 use risingwave_storage::dispatch_state_store;
 
@@ -42,6 +42,8 @@ impl ExecutorBuilder for SinkExecutorBuilder {
         let sink_desc = node.sink_desc.as_ref().unwrap();
         let sink_type = SinkType::from_proto(sink_desc.get_sink_type().unwrap());
         let sink_id = sink_desc.get_id().into();
+        let db_name = sink_desc.get_db_name().into();
+        let sink_from_name = sink_desc.get_sink_from_name().into();
         let properties = sink_desc.get_properties().clone();
         let pk_indices = sink_desc
             .downstream_pk
@@ -54,6 +56,19 @@ impl ExecutorBuilder for SinkExecutorBuilder {
             .into_iter()
             .map(ColumnCatalog::from)
             .collect_vec();
+        let sink_param = SinkParam {
+            sink_id,
+            properties,
+            columns: columns
+                .iter()
+                .filter(|col| !col.is_hidden)
+                .map(|col| col.column_desc.clone())
+                .collect(),
+            pk_indices,
+            sink_type,
+            db_name,
+            sink_from_name,
+        };
 
         match node.log_store_type() {
             // Default value is the normal in memory log store to be backward compatible with the
@@ -70,11 +85,8 @@ impl ExecutorBuilder for SinkExecutorBuilder {
                             vnode_bitmap: params.vnode_bitmap,
                             meta_client: params.env.meta_client(),
                         },
+                        sink_param,
                         columns,
-                        properties,
-                        pk_indices,
-                        sink_type,
-                        sink_id,
                         params.actor_context,
                         factory,
                     )
@@ -100,11 +112,8 @@ impl ExecutorBuilder for SinkExecutorBuilder {
                                 vnode_bitmap: params.vnode_bitmap,
                                 meta_client: params.env.meta_client(),
                             },
+                            sink_param,
                             columns,
-                            properties,
-                            pk_indices,
-                            sink_type,
-                            sink_id,
                             params.actor_context,
                             factory,
                         )
diff --git a/src/stream/src/from_proto/source.rs b/src/stream/src/from_proto/source.rs
index 47ee5c593f84a..f8487b98dc6a8 100644
--- a/src/stream/src/from_proto/source.rs
+++ b/src/stream/src/from_proto/source.rs
@@ -14,6 +14,8 @@
 
 use risingwave_common::catalog::{ColumnId, Field, Schema, TableId};
 use risingwave_common::types::DataType;
+use risingwave_common::util::sort_util::OrderType;
+use risingwave_connector::source::external::{ExternalTableType, SchemaTableName};
 use risingwave_connector::source::SourceCtrlOpts;
 use risingwave_pb::stream_plan::SourceNode;
 use risingwave_source::source_desc::SourceDescBuilder;
@@ -21,10 +23,11 @@ use risingwave_storage::panic_store::PanicStateStore;
 use tokio::sync::mpsc::unbounded_channel;
 
 use super::*;
+use crate::executor::external::ExternalStorageTable;
 use crate::executor::source::StreamSourceCore;
 use crate::executor::source_executor::SourceExecutor;
 use crate::executor::state_table_handler::SourceStateTableHandler;
-use crate::executor::FsSourceExecutor;
+use crate::executor::{CdcBackfillExecutor, FsSourceExecutor};
 
 const FS_CONNECTORS: &[&str] = &["s3"];
 pub struct SourceExecutorBuilder;
@@ -49,13 +52,14 @@ impl ExecutorBuilder for SourceExecutorBuilder {
         if let Some(source) = &node.source_inner {
             let source_id = TableId::new(source.source_id);
             let source_name = source.source_name.clone();
+            let source_info = source.get_info()?;
 
             let source_desc_builder = SourceDescBuilder::new(
                 source.columns.clone(),
                 params.env.source_metrics(),
                 source.row_id_index.map(|x| x as _),
                 source.properties.clone(),
-                source.get_info()?.clone(),
+                source_info.clone(),
                 params.env.connector_params(),
                 params.env.config().developer.connector_message_buffer_size,
                 // `pk_indices` is used to ensure that a message will be skipped instead of parsed
@@ -124,18 +128,67 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                     source_ctrl_opts,
                 )?))
             } else {
-                Ok(Box::new(SourceExecutor::new(
-                    params.actor_context,
-                    schema,
-                    params.pk_indices,
+                let source_exec = SourceExecutor::new(
+                    params.actor_context.clone(),
+                    schema.clone(),
+                    params.pk_indices.clone(),
                     Some(stream_source_core),
-                    params.executor_stats,
+                    params.executor_stats.clone(),
                     barrier_receiver,
                     system_params,
                     params.executor_id,
-                    source_ctrl_opts,
+                    source_ctrl_opts.clone(),
                     params.env.connector_params(),
-                )))
+                );
+
+                let table_type = ExternalTableType::from_properties(&source.properties);
+                if table_type.can_backfill() && let Some(table_desc) = source_info.upstream_table.clone() {
+                    let upstream_table_name = SchemaTableName::from_properties(&source.properties);
+                    let pk_indices = table_desc
+                        .pk
+                        .iter()
+                        .map(|k| k.column_index as usize)
+                        .collect_vec();
+
+                    let order_types = table_desc
+                        .pk
+                        .iter()
+                        .map(|desc| OrderType::from_protobuf(desc.get_order_type().unwrap()))
+                        .collect_vec();
+
+                    let table_reader = table_type.create_table_reader(source.properties.clone(), schema.clone())?;
+                    let external_table = ExternalStorageTable::new(
+                         TableId::new(source.source_id),
+                        upstream_table_name,
+                        table_reader,
+                        schema.clone(),
+                        order_types,
+                        pk_indices.clone(),
+                         (0..table_desc.columns.len()).collect_vec(),
+                    );
+
+                    // use the state table from source to store the backfill state (may refactor in future)
+                    let source_state_handler = SourceStateTableHandler::from_table_catalog(
+                        source.state_table.as_ref().unwrap(),
+                        store.clone(),
+                    ).await;
+                    let cdc_backfill = CdcBackfillExecutor::new(
+                        params.actor_context.clone(),
+                        external_table,
+                        Box::new(source_exec),
+                        (0..source.columns.len()).collect_vec(),    // eliminate the last column (_rw_offset)
+                        None,
+                        schema.clone(),
+                        pk_indices,
+                        params.executor_stats,
+                        source_state_handler,
+                        source_ctrl_opts.chunk_size
+                    );
+                    Ok(Box::new(cdc_backfill))
+
+                } else {
+                    Ok(Box::new(source_exec))
+                }
             }
         } else {
             // If there is no external stream source, then no data should be persisted. We pass a
diff --git a/src/stream/src/lib.rs b/src/stream/src/lib.rs
index 55b675d51e858..e2f53aa9df1c5 100644
--- a/src/stream/src/lib.rs
+++ b/src/stream/src/lib.rs
@@ -44,6 +44,7 @@
 #![feature(is_sorted)]
 #![feature(btree_cursors)]
 #![feature(assert_matches)]
+#![feature(async_fn_in_trait)]
 
 #[macro_use]
 extern crate tracing;
diff --git a/src/stream/src/task/barrier_manager.rs b/src/stream/src/task/barrier_manager.rs
index 25572a4a8d17b..5581a8529c067 100644
--- a/src/stream/src/task/barrier_manager.rs
+++ b/src/stream/src/task/barrier_manager.rs
@@ -101,7 +101,11 @@ impl LocalBarrierManager {
 
     /// Register sender for source actors, used to send barriers.
     pub fn register_sender(&mut self, actor_id: ActorId, sender: UnboundedSender<Barrier>) {
-        tracing::trace!(actor_id = actor_id, "register sender");
+        tracing::trace!(
+            target: "events::stream::barrier::manager",
+            actor_id = actor_id,
+            "register sender"
+        );
         self.senders.entry(actor_id).or_default().push(sender);
     }
 
@@ -129,6 +133,7 @@ impl LocalBarrierManager {
         };
         let to_collect: HashSet<ActorId> = actor_ids_to_collect.into_iter().collect();
         trace!(
+            target: "events::stream::barrier::manager::send",
             "send barrier {:?}, senders = {:?}, actor_ids_to_collect = {:?}",
             barrier,
             to_send,
@@ -167,7 +172,11 @@ impl LocalBarrierManager {
 
         // Actors to stop should still accept this barrier, but won't get sent to in next times.
         if let Some(actors) = barrier.all_stop_actors() {
-            trace!("remove actors {:?} from senders", actors);
+            trace!(
+                target: "events::stream::barrier::manager",
+                "remove actors {:?} from senders",
+                actors
+            );
             for actor in actors {
                 self.senders.remove(actor);
             }
diff --git a/src/stream/src/task/barrier_manager/managed_state.rs b/src/stream/src/task/barrier_manager/managed_state.rs
index 5599904b99259..c438272033831 100644
--- a/src/stream/src/task/barrier_manager/managed_state.rs
+++ b/src/stream/src/task/barrier_manager/managed_state.rs
@@ -25,6 +25,7 @@ use tokio::sync::oneshot;
 use super::progress::ChainState;
 use super::CollectResult;
 use crate::error::{StreamError, StreamResult};
+use crate::executor::monitor::GLOBAL_STREAMING_METRICS;
 use crate::executor::Barrier;
 use crate::task::ActorId;
 
@@ -84,85 +85,73 @@ impl ManagedBarrierState {
 
     /// Notify if we have collected barriers from all actor ids. The state must be `Issued`.
     fn may_notify(&mut self, curr_epoch: u64) {
-        let to_notify = match self.epoch_barrier_state_map.get(&curr_epoch) {
-            Some(BarrierState {
-                inner:
-                    ManagedBarrierStateInner::Issued {
-                        remaining_actors, ..
-                    },
-                ..
-            }) => remaining_actors.is_empty(),
-            _ => unreachable!(),
-        };
+        // Report if there's progress on the earliest in-flight barrier.
+        if self.epoch_barrier_state_map.keys().next() == Some(&curr_epoch) {
+            if let Some(metrics) = GLOBAL_STREAMING_METRICS.get() {
+                metrics.barrier_manager_progress.inc();
+            }
+        }
 
-        if to_notify {
-            while let Some((
-                _,
-                BarrierState {
-                    inner: barrier_inner,
-                    ..
-                },
-            )) = self.epoch_barrier_state_map.first_key_value()
-            {
-                match barrier_inner {
-                    ManagedBarrierStateInner::Issued {
-                        remaining_actors, ..
-                    } => {
-                        if !remaining_actors.is_empty() {
-                            break;
-                        }
-                    }
-                    _ => break,
-                }
-                let (epoch, barrier_state) = self.epoch_barrier_state_map.pop_first().unwrap();
-                let create_mview_progress = self
-                    .create_mview_progress
-                    .remove(&epoch)
-                    .unwrap_or_default()
-                    .into_iter()
-                    .map(|(actor, state)| CreateMviewProgress {
-                        chain_actor_id: actor,
-                        done: matches!(state, ChainState::Done),
-                        consumed_epoch: match state {
-                            ChainState::ConsumingUpstream(consumed_epoch, _) => consumed_epoch,
-                            ChainState::Done => epoch,
-                        },
-                        consumed_rows: match state {
-                            ChainState::ConsumingUpstream(_, consumed_rows) => consumed_rows,
-                            ChainState::Done => 0,
-                        },
-                    })
-                    .collect();
+        while let Some(entry) = self.epoch_barrier_state_map.first_entry() {
+            let to_notify = matches!(
+                &entry.get().inner,
+                ManagedBarrierStateInner::Issued {
+                    remaining_actors, ..
+                } if remaining_actors.is_empty(),
+            );
 
-                let kind = barrier_state.kind;
-                match kind {
-                    BarrierKind::Unspecified => unreachable!(),
-                    BarrierKind::Initial => tracing::info!(
-                        epoch = barrier_state.prev_epoch,
-                        "ignore sealing data for the first barrier"
-                    ),
-                    BarrierKind::Barrier | BarrierKind::Checkpoint => {
-                        dispatch_state_store!(&self.state_store, state_store, {
-                            state_store.seal_epoch(barrier_state.prev_epoch, kind.is_checkpoint());
-                        });
-                    }
+            if !to_notify {
+                break;
+            }
+
+            let (epoch, barrier_state) = entry.remove_entry();
+            let create_mview_progress = self
+                .create_mview_progress
+                .remove(&epoch)
+                .unwrap_or_default()
+                .into_iter()
+                .map(|(actor, state)| CreateMviewProgress {
+                    chain_actor_id: actor,
+                    done: matches!(state, ChainState::Done),
+                    consumed_epoch: match state {
+                        ChainState::ConsumingUpstream(consumed_epoch, _) => consumed_epoch,
+                        ChainState::Done => epoch,
+                    },
+                    consumed_rows: match state {
+                        ChainState::ConsumingUpstream(_, consumed_rows) => consumed_rows,
+                        ChainState::Done => 0,
+                    },
+                })
+                .collect();
+
+            let kind = barrier_state.kind;
+            match kind {
+                BarrierKind::Unspecified => unreachable!(),
+                BarrierKind::Initial => tracing::info!(
+                    epoch = barrier_state.prev_epoch,
+                    "ignore sealing data for the first barrier"
+                ),
+                BarrierKind::Barrier | BarrierKind::Checkpoint => {
+                    dispatch_state_store!(&self.state_store, state_store, {
+                        state_store.seal_epoch(barrier_state.prev_epoch, kind.is_checkpoint());
+                    });
                 }
+            }
 
-                match barrier_state.inner {
-                    ManagedBarrierStateInner::Issued {
-                        collect_notifier, ..
-                    } => {
-                        // Notify about barrier finishing.
-                        let result = CollectResult {
-                            create_mview_progress,
-                            kind,
-                        };
-                        if collect_notifier.unwrap().send(Ok(result)).is_err() {
-                            warn!("failed to notify barrier collection with epoch {}", epoch)
-                        }
+            match barrier_state.inner {
+                ManagedBarrierStateInner::Issued {
+                    collect_notifier, ..
+                } => {
+                    // Notify about barrier finishing.
+                    let result = CollectResult {
+                        create_mview_progress,
+                        kind,
+                    };
+                    if collect_notifier.unwrap().send(Ok(result)).is_err() {
+                        warn!("failed to notify barrier collection with epoch {}", epoch)
                     }
-                    _ => unreachable!(),
                 }
+                _ => unreachable!(),
             }
         }
     }
@@ -205,7 +194,7 @@ impl ManagedBarrierState {
     /// Collect a `barrier` from the actor with `actor_id`.
     pub(super) fn collect(&mut self, actor_id: ActorId, barrier: &Barrier) {
         tracing::trace!(
-            target: "events::stream::barrier::collect_barrier",
+            target: "events::stream::barrier::manager::collect",
             "collect_barrier: epoch = {}, actor_id = {}, state = {:#?}",
             barrier.epoch.curr,
             actor_id,
diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs
index dfecd4d616694..2abc8212e2984 100644
--- a/src/stream/src/task/stream_manager.rs
+++ b/src/stream/src/task/stream_manager.rs
@@ -27,7 +27,7 @@ use itertools::Itertools;
 use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::{Field, Schema};
-use risingwave_common::config::StreamingConfig;
+use risingwave_common::config::{MetricLevel, StreamingConfig};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::runtime::BackgroundShutdownRuntime;
 use risingwave_hummock_sdk::LocalSstableInfo;
@@ -694,61 +694,63 @@ impl LocalStreamManagerCore {
             };
             self.handles.insert(actor_id, handle);
 
-            let actor_id_str = actor_id.to_string();
-
-            let metrics = self.streaming_metrics.clone();
-            let actor_monitor_task = self.runtime.spawn(async move {
-                loop {
-                    let task_metrics = monitor.cumulative();
-                    metrics
-                        .actor_execution_time
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_poll_duration.as_secs_f64());
-                    metrics
-                        .actor_fast_poll_duration
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_fast_poll_duration.as_secs_f64());
-                    metrics
-                        .actor_fast_poll_cnt
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_fast_poll_count as i64);
-                    metrics
-                        .actor_slow_poll_duration
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_slow_poll_duration.as_secs_f64());
-                    metrics
-                        .actor_slow_poll_cnt
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_slow_poll_count as i64);
-                    metrics
-                        .actor_poll_duration
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_poll_duration.as_secs_f64());
-                    metrics
-                        .actor_poll_cnt
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_poll_count as i64);
-                    metrics
-                        .actor_idle_duration
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_idle_duration.as_secs_f64());
-                    metrics
-                        .actor_idle_cnt
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_idled_count as i64);
-                    metrics
-                        .actor_scheduled_duration
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_scheduled_duration.as_secs_f64());
-                    metrics
-                        .actor_scheduled_cnt
-                        .with_label_values(&[&actor_id_str])
-                        .set(task_metrics.total_scheduled_count as i64);
-                    tokio::time::sleep(Duration::from_secs(1)).await;
-                }
-            });
-            self.actor_monitor_tasks
-                .insert(actor_id, actor_monitor_task);
+            if self.streaming_metrics.level >= MetricLevel::Debug {
+                tracing::info!("Tokio metrics are enabled because metrics_level >= Debug");
+                let actor_id_str = actor_id.to_string();
+                let metrics = self.streaming_metrics.clone();
+                let actor_monitor_task = self.runtime.spawn(async move {
+                    loop {
+                        let task_metrics = monitor.cumulative();
+                        metrics
+                            .actor_execution_time
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_poll_duration.as_secs_f64());
+                        metrics
+                            .actor_fast_poll_duration
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_fast_poll_duration.as_secs_f64());
+                        metrics
+                            .actor_fast_poll_cnt
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_fast_poll_count as i64);
+                        metrics
+                            .actor_slow_poll_duration
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_slow_poll_duration.as_secs_f64());
+                        metrics
+                            .actor_slow_poll_cnt
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_slow_poll_count as i64);
+                        metrics
+                            .actor_poll_duration
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_poll_duration.as_secs_f64());
+                        metrics
+                            .actor_poll_cnt
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_poll_count as i64);
+                        metrics
+                            .actor_idle_duration
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_idle_duration.as_secs_f64());
+                        metrics
+                            .actor_idle_cnt
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_idled_count as i64);
+                        metrics
+                            .actor_scheduled_duration
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_scheduled_duration.as_secs_f64());
+                        metrics
+                            .actor_scheduled_cnt
+                            .with_label_values(&[&actor_id_str])
+                            .set(task_metrics.total_scheduled_count as i64);
+                        tokio::time::sleep(Duration::from_secs(1)).await;
+                    }
+                });
+                self.actor_monitor_tasks
+                    .insert(actor_id, actor_monitor_task);
+            }
         }
 
         Ok(())
diff --git a/src/stream/tests/integration_tests/over_window.rs b/src/stream/tests/integration_tests/over_window.rs
index 4b25c39a0554c..4b7b53aaae31b 100644
--- a/src/stream/tests/integration_tests/over_window.rs
+++ b/src/stream/tests/integration_tests/over_window.rs
@@ -12,10 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::session_config::OverWindowCachePolicy;
 use risingwave_expr::agg::{AggArgs, AggKind};
 use risingwave_expr::function::window::{
     Frame, FrameBound, FrameExclusion, WindowFuncCall, WindowFuncKind,
 };
+use risingwave_stream::executor::monitor::StreamingMetrics;
 use risingwave_stream::executor::{OverWindowExecutor, OverWindowExecutorArgs};
 
 use crate::prelude::*;
@@ -77,7 +79,9 @@ async fn create_executor<S: StateStore>(
         order_key_order_types,
         state_table,
         watermark_epoch: Arc::new(AtomicU64::new(0)),
+        metrics: Arc::new(StreamingMetrics::unused()),
         chunk_size: 1024,
+        cache_policy: OverWindowCachePolicy::Recent,
     });
     (tx, executor.boxed().execute())
 }
diff --git a/src/stream/tests/integration_tests/project_set.rs b/src/stream/tests/integration_tests/project_set.rs
index c98f7d65e4c32..bf1354c25b83b 100644
--- a/src/stream/tests/integration_tests/project_set.rs
+++ b/src/stream/tests/integration_tests/project_set.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use multimap::MultiMap;
 use risingwave_expr::table_function::repeat;
 use risingwave_stream::executor::ProjectSetExecutor;
 
@@ -29,15 +30,24 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) {
     let (tx, source) = MockSource::channel(schema, PkIndices::new());
 
     let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)");
+    let test_expr_watermark = build_from_pretty("(add:int8 $0:int8 1:int8)");
     let tf1 = repeat(build_from_pretty("1:int4"), 1);
     let tf2 = repeat(build_from_pretty("2:int4"), 2);
 
     let project_set = Box::new(ProjectSetExecutor::new(
+        ActorContext::create(123),
         Box::new(source),
         vec![],
-        vec![test_expr.into(), tf1.into(), tf2.into()],
+        vec![
+            test_expr.into(),
+            test_expr_watermark.into(),
+            tf1.into(),
+            tf2.into(),
+        ],
         1,
         CHUNK_SIZE,
+        MultiMap::from_iter(std::iter::once((0, 1))),
+        vec![],
     ));
     (tx, project_set.execute())
 }
@@ -52,6 +62,7 @@ async fn test_project_set() {
         + 2 5
         + 3 6",
     ));
+    tx.push_int64_watermark(0, 3);
     tx.push_chunk(StreamChunk::from_pretty(
         " I I
         + 7 8
@@ -62,21 +73,24 @@ async fn test_project_set() {
         &mut project_set,
         expect_test::expect![[r#"
             - !chunk |-
-              +---+---+---+---+---+
-              | + | 0 | 5 | 1 | 2 |
-              | + | 1 | 5 |   | 2 |
-              | + | 0 | 7 | 1 | 2 |
-              | + | 1 | 7 |   | 2 |
-              | + | 0 | 9 | 1 | 2 |
-              | + | 1 | 9 |   | 2 |
-              +---+---+---+---+---+
+              +---+---+---+---+---+---+
+              | + | 0 | 5 | 2 | 1 | 2 |
+              | + | 1 | 5 | 2 |   | 2 |
+              | + | 0 | 7 | 3 | 1 | 2 |
+              | + | 1 | 7 | 3 |   | 2 |
+              | + | 0 | 9 | 4 | 1 | 2 |
+              | + | 1 | 9 | 4 |   | 2 |
+              +---+---+---+---+---+---+
+            - !watermark
+              col_idx: 2
+              val: '4'
             - !chunk |-
-              +---+---+----+---+---+
-              | + | 0 | 15 | 1 | 2 |
-              | + | 1 | 15 |   | 2 |
-              | - | 0 | 9  | 1 | 2 |
-              | - | 1 | 9  |   | 2 |
-              +---+---+----+---+---+
+              +---+---+----+---+---+---+
+              | + | 0 | 15 | 8 | 1 | 2 |
+              | + | 1 | 15 | 8 |   | 2 |
+              | - | 0 | 9  | 4 | 1 | 2 |
+              | - | 1 | 9  | 4 |   | 2 |
+              +---+---+----+---+---+---+
         "#]],
         SnapshotOptions::default(),
     );
diff --git a/src/stream/tests/integration_tests/snapshot.rs b/src/stream/tests/integration_tests/snapshot.rs
index 0aaa948a5182a..272a0e20bcf47 100644
--- a/src/stream/tests/integration_tests/snapshot.rs
+++ b/src/stream/tests/integration_tests/snapshot.rs
@@ -154,7 +154,7 @@ where
             }
             SnapshotEvent::Chunk(chunk_str) => {
                 let chunk = StreamChunk::from_pretty(chunk_str);
-                *chunk_str = chunk.to_pretty_string();
+                *chunk_str = chunk.to_pretty().to_string();
                 tx.push_chunk(chunk);
             }
             SnapshotEvent::Watermark { col_idx, val } => tx.push_watermark(
@@ -191,10 +191,10 @@ fn run_until_pending(
                 if options.sort_chunk {
                     chunk = chunk.sort_rows();
                 }
-                let mut output = chunk.to_pretty_string();
+                let mut output = chunk.to_pretty().to_string();
                 if options.include_applied_result {
                     let applied = store.apply_chunk(&chunk);
-                    output += &format!("\napplied result:\n{}", applied.to_pretty_string());
+                    output += &format!("\napplied result:\n{}", applied.to_pretty());
                 }
                 SnapshotEvent::Chunk(output)
             }
diff --git a/src/tests/compaction_test/src/compaction_test_runner.rs b/src/tests/compaction_test/src/compaction_test_runner.rs
index 989745520fe6e..db248ad788fba 100644
--- a/src/tests/compaction_test/src/compaction_test_runner.rs
+++ b/src/tests/compaction_test/src/compaction_test_runner.rs
@@ -631,13 +631,9 @@ async fn open_hummock_iters(
                 range.clone(),
                 epoch,
                 ReadOptions {
-                    prefix_hint: None,
                     table_id: TableId { table_id },
-                    retention_seconds: None,
-                    ignore_range_tombstone: false,
-                    read_version_from_backup: false,
-                    prefetch_options: Default::default(),
                     cache_policy: CachePolicy::Fill(CachePriority::High),
+                    ..Default::default()
                 },
             )
             .await?;
diff --git a/src/tests/compaction_test/src/delete_range_runner.rs b/src/tests/compaction_test/src/delete_range_runner.rs
index b55c6c90448b5..683cff8fac45d 100644
--- a/src/tests/compaction_test/src/delete_range_runner.rs
+++ b/src/tests/compaction_test/src/delete_range_runner.rs
@@ -30,6 +30,7 @@ use risingwave_common::catalog::TableId;
 use risingwave_common::config::{extract_storage_memory_config, load_config, NoOverride, RwConfig};
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_test::get_notification_client_for_test;
+use risingwave_hummock_test::local_state_store_test_utils::LocalStateStoreTestExt;
 use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder;
 use risingwave_meta::hummock::test_utils::setup_compute_env_with_config;
 use risingwave_meta::hummock::MockHummockMetaClient;
@@ -41,6 +42,7 @@ use risingwave_pb::meta::SystemParams;
 use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::filter_key_extractor::{
     FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor,
+    RpcFilterKeyExtractorManager,
 };
 use risingwave_storage::hummock::compactor::{
     start_compactor, CompactionExecutor, CompactorContext,
@@ -212,13 +214,19 @@ async fn compaction_test(
         sstable_store.clone(),
         meta_client.clone(),
         get_notification_client_for_test(env, hummock_manager_ref.clone(), worker_node),
-        Arc::new(FilterKeyExtractorManager::default()),
+        Arc::new(RpcFilterKeyExtractorManager::default()),
         state_store_metrics.clone(),
         compactor_metrics.clone(),
     )
     .await?;
     let sstable_object_id_manager = store.sstable_object_id_manager().clone();
-    let filter_key_extractor_manager = store.filter_key_extractor_manager().clone();
+    let filter_key_extractor_manager = match store.filter_key_extractor_manager().clone() {
+        FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+            rpc_filter_key_extractor_manager,
+        ) => rpc_filter_key_extractor_manager,
+        FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
+    };
+
     filter_key_extractor_manager.update(
         1,
         Arc::new(FilterKeyExtractorImpl::FullKey(
@@ -391,7 +399,7 @@ impl NormalState {
     async fn new(hummock: &HummockStorage, table_id: u32, epoch: u64) -> Self {
         let table_id = TableId::new(table_id);
         let mut storage = hummock.new_local(NewLocalOptions::for_test(table_id)).await;
-        storage.init(epoch);
+        storage.init_for_test(epoch).await.unwrap();
         Self { storage, table_id }
     }
 
@@ -560,28 +568,28 @@ fn run_compactor_thread(
     storage_opts: Arc<StorageOpts>,
     sstable_store: SstableStoreRef,
     meta_client: Arc<MockHummockMetaClient>,
-    filter_key_extractor_manager: Arc<FilterKeyExtractorManager>,
+    filter_key_extractor_manager: Arc<RpcFilterKeyExtractorManager>,
     sstable_object_id_manager: Arc<SstableObjectIdManager>,
     compactor_metrics: Arc<CompactorMetrics>,
 ) -> (
     tokio::task::JoinHandle<()>,
     tokio::sync::oneshot::Sender<()>,
 ) {
-    let compactor_context = Arc::new(CompactorContext {
+    let compactor_context = CompactorContext {
         storage_opts,
-        hummock_meta_client: meta_client,
         sstable_store,
         compactor_metrics,
         is_share_buffer_compact: false,
         compaction_executor: Arc::new(CompactionExecutor::new(None)),
-        filter_key_extractor_manager,
+        filter_key_extractor_manager: FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
+            filter_key_extractor_manager,
+        ),
         memory_limiter: MemoryLimiter::unlimit(),
-        sstable_object_id_manager,
         task_progress_manager: Default::default(),
         await_tree_reg: None,
         running_task_count: Arc::new(AtomicU32::new(0)),
-    });
-    start_compactor(compactor_context)
+    };
+    start_compactor(compactor_context, meta_client, sstable_object_id_manager)
 }
 
 #[cfg(test)]
diff --git a/src/tests/e2e_extended_mode/Cargo.toml b/src/tests/e2e_extended_mode/Cargo.toml
index 274fdcf8d4431..7e89105287602 100644
--- a/src/tests/e2e_extended_mode/Cargo.toml
+++ b/src/tests/e2e_extended_mode/Cargo.toml
@@ -18,7 +18,7 @@ anyhow = { version = "1", features = ["backtrace"] }
 chrono = { version = "0.4", features = ['serde'] }
 clap = { version = "4", features = ["derive"] }
 pg_interval = "0.4"
-rust_decimal ={ version = "1.31", features = ["db-postgres"] }
+rust_decimal ={ version = "1.32", features = ["db-postgres"] }
 tokio = { version = "0.2.23", package = "madsim-tokio", features = ["rt", "macros","rt-multi-thread"] }
 tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] }
 tracing = "0.1"
diff --git a/src/tests/e2e_extended_mode/src/test.rs b/src/tests/e2e_extended_mode/src/test.rs
index 0ca5098d33a6a..21fcf1f146787 100644
--- a/src/tests/e2e_extended_mode/src/test.rs
+++ b/src/tests/e2e_extended_mode/src/test.rs
@@ -187,7 +187,7 @@ impl TestSuite {
             );
         }
 
-        let timestamptz = DateTime::<Utc>::from_utc(
+        let timestamptz = DateTime::<Utc>::from_naive_utc_and_offset(
             NaiveDate::from_ymd_opt(2022, 1, 1)
                 .unwrap()
                 .and_hms_opt(10, 0, 0)
diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml
index 0b5534fc06b01..dd21ac0ac6949 100644
--- a/src/tests/simulation/Cargo.toml
+++ b/src/tests/simulation/Cargo.toml
@@ -14,6 +14,7 @@ normal = ["serde"]
 anyhow = "1.0"
 async-trait = "0.1"
 aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3" }
+cfg-or-panic = "0.1"
 clap = { version = "4", features = ["derive"] }
 console = "0.15"
 etcd-client = { workspace = true }
@@ -40,7 +41,7 @@ risingwave_pb = { workspace = true }
 risingwave_rpc_client = { workspace = true }
 risingwave_sqlparser = { workspace = true }
 risingwave_sqlsmith = { workspace = true }
-serde = "1.0.185"
+serde = "1.0.188"
 serde_derive = "1.0.183"
 serde_json = "1.0.105"
 sqllogictest = "0.15.2"
diff --git a/src/tests/simulation/src/backfill.toml b/src/tests/simulation/src/backfill.toml
new file mode 100644
index 0000000000000..7f4b6b8da0497
--- /dev/null
+++ b/src/tests/simulation/src/backfill.toml
@@ -0,0 +1,13 @@
+[server]
+telemetry_enabled = false
+metrics_level = "Disabled"
+
+[streaming.developer]
+stream_chunk_size = 1
+
+[system]
+# NOTE(kwannoel): If can't reproduce it, set to a lower number.
+# This will throttle snapshot read.
+# barrier_interval_ms = 1
+barrier_interval_ms = 50
+max_concurrent_creating_streaming_jobs = 0
diff --git a/src/tests/simulation/src/client.rs b/src/tests/simulation/src/client.rs
index 498aaa2500edf..089d67bceeeab 100644
--- a/src/tests/simulation/src/client.rs
+++ b/src/tests/simulation/src/client.rs
@@ -64,7 +64,7 @@ impl<'a, 'b> SetStmtsIterator<'a, 'b> {
 
 impl SetStmts {
     fn push(&mut self, sql: &str) {
-        let ast = Parser::parse_sql(&sql).expect("a set statement should be parsed successfully");
+        let ast = Parser::parse_sql(sql).expect("a set statement should be parsed successfully");
         match ast
             .into_iter()
             .exactly_one()
@@ -134,7 +134,7 @@ impl RisingWave {
             .simple_query("SET RW_IMPLICIT_FLUSH TO true;")
             .await?;
         // replay all SET statements
-        for stmt in SetStmtsIterator::new(&set_stmts) {
+        for stmt in SetStmtsIterator::new(set_stmts) {
             client.simple_query(&stmt).await?;
         }
         Ok((client, task))
diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs
index 9c9b426a04001..f375eeac4cc85 100644
--- a/src/tests/simulation/src/cluster.rs
+++ b/src/tests/simulation/src/cluster.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg_attr(not(madsim), allow(unused_imports))]
+
 use std::collections::HashMap;
 use std::future::Future;
 use std::io::Write;
@@ -20,17 +22,20 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{anyhow, bail, Result};
+use cfg_or_panic::cfg_or_panic;
 use clap::Parser;
 use futures::channel::{mpsc, oneshot};
 use futures::future::join_all;
 use futures::{SinkExt, StreamExt};
 use itertools::Itertools;
-use madsim::net::ipvs::*;
+#[cfg(madsim)]
 use madsim::runtime::{Handle, NodeHandle};
 use rand::seq::IteratorRandom;
 use rand::Rng;
 use risingwave_pb::common::WorkerNode;
 use sqllogictest::AsyncDB;
+#[cfg(not(madsim))]
+use tokio::runtime::Handle;
 
 use crate::client::RisingWave;
 
@@ -116,7 +121,7 @@ impl Configuration {
         let config_path = {
             let mut file =
                 tempfile::NamedTempFile::new().expect("failed to create temp config file");
-            file.write_all(include_bytes!("../../../../src/config/backfill.toml"))
+            file.write_all(include_bytes!("backfill.toml"))
                 .expect("failed to write config file");
             file.into_temp_path()
         };
@@ -153,7 +158,9 @@ impl Configuration {
 pub struct Cluster {
     config: Configuration,
     handle: Handle,
+    #[cfg(madsim)]
     pub(crate) client: NodeHandle,
+    #[cfg(madsim)]
     pub(crate) ctl: NodeHandle,
 }
 
@@ -161,7 +168,10 @@ impl Cluster {
     /// Start a RisingWave cluster for testing.
     ///
     /// This function should be called exactly once in a test.
+    #[cfg_or_panic(madsim)]
     pub async fn start(conf: Configuration) -> Result<Self> {
+        use madsim::net::ipvs::*;
+
         let handle = madsim::runtime::Handle::current();
         println!("seed = {}", handle.seed());
         println!("{:#?}", conf);
@@ -361,6 +371,7 @@ impl Cluster {
     }
 
     /// Start a SQL session on the client node.
+    #[cfg_or_panic(madsim)]
     pub fn start_session(&mut self) -> Session {
         let (query_tx, mut query_rx) = mpsc::channel::<SessionRequest>(0);
 
@@ -404,6 +415,7 @@ impl Cluster {
     }
 
     /// Run a future on the client node.
+    #[cfg_or_panic(madsim)]
     pub async fn run_on_client<F>(&self, future: F) -> F::Output
     where
         F: Future + Send + 'static,
@@ -433,7 +445,7 @@ impl Cluster {
         timeout: Duration,
     ) -> Result<String> {
         let fut = async move {
-            let mut interval = madsim::time::interval(interval);
+            let mut interval = tokio::time::interval(interval);
             loop {
                 interval.tick().await;
                 let result = self.run(sql.clone()).await?;
@@ -443,7 +455,7 @@ impl Cluster {
             }
         };
 
-        match madsim::time::timeout(timeout, fut).await {
+        match tokio::time::timeout(timeout, fut).await {
             Ok(r) => Ok(r?),
             Err(_) => bail!("wait_until timeout"),
         }
@@ -460,7 +472,8 @@ impl Cluster {
             .await
     }
 
-    /// Kill some nodes and restart them in 2s + restart_delay_secs with a probability of 0.1.
+    /// Generate a list of random worker nodes to kill by `opts`, then call `kill_nodes` to kill and
+    /// restart them.
     pub async fn kill_node(&self, opts: &KillOpts) {
         let mut nodes = vec![];
         if opts.kill_meta {
@@ -515,11 +528,24 @@ impl Cluster {
                 nodes.push(format!("compactor-{}", i));
             }
         }
-        join_all(nodes.iter().map(|name| async move {
+
+        self.kill_nodes(nodes, opts.restart_delay_secs).await
+    }
+
+    /// Kill the given nodes by their names and restart them in 2s + restart_delay_secs with a
+    /// probability of 0.1.
+    #[cfg_or_panic(madsim)]
+    pub async fn kill_nodes(
+        &self,
+        nodes: impl IntoIterator<Item = impl AsRef<str>>,
+        restart_delay_secs: u32,
+    ) {
+        join_all(nodes.into_iter().map(|name| async move {
+            let name = name.as_ref();
             let t = rand::thread_rng().gen_range(Duration::from_secs(0)..Duration::from_secs(1));
             tokio::time::sleep(t).await;
             tracing::info!("kill {name}");
-            madsim::runtime::Handle::current().kill(name);
+            Handle::current().kill(name);
 
             let mut t =
                 rand::thread_rng().gen_range(Duration::from_secs(0)..Duration::from_secs(1));
@@ -527,16 +553,17 @@ impl Cluster {
             // so that the node is expired and removed from the cluster
             if rand::thread_rng().gen_bool(0.1) {
                 // max_heartbeat_interval_secs = 15
-                t += Duration::from_secs(opts.restart_delay_secs as u64);
+                t += Duration::from_secs(restart_delay_secs as u64);
             }
             tokio::time::sleep(t).await;
             tracing::info!("restart {name}");
-            madsim::runtime::Handle::current().restart(name);
+            Handle::current().restart(name);
         }))
         .await;
     }
 
     /// Create a node for kafka producer and prepare data.
+    #[cfg_or_panic(madsim)]
     pub async fn create_kafka_producer(&self, datadir: &str) {
         self.handle
             .create_node()
@@ -552,6 +579,7 @@ impl Cluster {
     }
 
     /// Create a kafka topic.
+    #[cfg_or_panic(madsim)]
     pub fn create_kafka_topics(&self, topics: HashMap<String, i32>) {
         self.handle
             .create_node()
@@ -570,6 +598,7 @@ impl Cluster {
     }
 
     /// Graceful shutdown all RisingWave nodes.
+    #[cfg_or_panic(madsim)]
     pub async fn graceful_shutdown(&self) {
         let mut nodes = vec![];
         let mut metas = vec![];
@@ -592,12 +621,12 @@ impl Cluster {
         for node in &nodes {
             self.handle.send_ctrl_c(node);
         }
-        madsim::time::sleep(waiting_time).await;
+        tokio::time::sleep(waiting_time).await;
         // shutdown metas
         for meta in &metas {
             self.handle.send_ctrl_c(meta);
         }
-        madsim::time::sleep(waiting_time).await;
+        tokio::time::sleep(waiting_time).await;
 
         // check all nodes are exited
         for node in nodes.iter().chain(metas.iter()) {
diff --git a/src/tests/simulation/src/ctl_ext.rs b/src/tests/simulation/src/ctl_ext.rs
index feb2b40be7986..850f881031e73 100644
--- a/src/tests/simulation/src/ctl_ext.rs
+++ b/src/tests/simulation/src/ctl_ext.rs
@@ -12,18 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg_attr(not(madsim), expect(unused_imports))]
+
 use std::collections::{HashMap, HashSet};
 use std::fmt::Write;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Result};
+use cfg_or_panic::cfg_or_panic;
 use clap::Parser;
 use itertools::Itertools;
-use madsim::rand::thread_rng;
 use rand::seq::{IteratorRandom, SliceRandom};
-use rand::Rng;
+use rand::{thread_rng, Rng};
 use risingwave_common::hash::ParallelUnitId;
-use risingwave_pb::common::{HostAddress, WorkerNode};
 use risingwave_pb::meta::get_reschedule_plan_request::PbPolicy;
 use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
 use risingwave_pb::meta::table_fragments::PbFragment;
@@ -219,6 +220,7 @@ impl Fragment {
 
 impl Cluster {
     /// Locate fragments that satisfy all the predicates.
+    #[cfg_or_panic(madsim)]
     pub async fn locate_fragments(
         &mut self,
         predicates: impl IntoIterator<Item = BoxedPredicate>,
@@ -291,6 +293,7 @@ impl Cluster {
         self.locate_one_fragment([predicate::id(id)]).await
     }
 
+    #[cfg_or_panic(madsim)]
     pub async fn get_cluster_info(&self) -> Result<GetClusterInfoResponse> {
         let response = self
             .ctl
@@ -305,6 +308,7 @@ impl Cluster {
     }
 
     // update node schedulability
+    #[cfg_or_panic(madsim)]
     async fn update_worker_node_schedulability(
         &self,
         worker_ids: Vec<u32>,
@@ -345,11 +349,12 @@ impl Cluster {
         self.reschedule_helper(plan, false).await
     }
 
-    /// Same as reschedule, but resolve the no_shuffle upstream
+    /// Same as reschedule, but resolve the no-shuffle upstream
     pub async fn reschedule_resolve_no_shuffle(&mut self, plan: impl Into<String>) -> Result<()> {
         self.reschedule_helper(plan, true).await
     }
 
+    #[cfg_or_panic(madsim)]
     async fn reschedule_helper(
         &mut self,
         plan: impl Into<String>,
@@ -394,6 +399,33 @@ impl Cluster {
         Ok(())
     }
 
+    /// Pause all data sources in the cluster.
+    #[cfg_or_panic(madsim)]
+    pub async fn pause(&mut self) -> Result<()> {
+        self.ctl
+            .spawn(async move {
+                let opts = risingwave_ctl::CliOpts::parse_from(["ctl", "meta", "pause"]);
+                risingwave_ctl::start(opts).await
+            })
+            .await??;
+
+        Ok(())
+    }
+
+    /// Resume all data sources in the cluster.
+    #[cfg_or_panic(madsim)]
+    pub async fn resume(&mut self) -> Result<()> {
+        self.ctl
+            .spawn(async move {
+                let opts = risingwave_ctl::CliOpts::parse_from(["ctl", "meta", "resume"]);
+                risingwave_ctl::start(opts).await
+            })
+            .await??;
+
+        Ok(())
+    }
+
+    #[cfg_or_panic(madsim)]
     pub async fn get_reschedule_plan(&self, policy: PbPolicy) -> Result<GetReschedulePlanResponse> {
         let revision = self
             .ctl
diff --git a/src/tests/simulation/src/kafka.rs b/src/tests/simulation/src/kafka.rs
index b62072961e629..0ab4829d9e1b8 100644
--- a/src/tests/simulation/src/kafka.rs
+++ b/src/tests/simulation/src/kafka.rs
@@ -15,9 +15,10 @@
 use std::collections::HashMap;
 use std::time::SystemTime;
 
+use itertools::Either;
 use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
 use rdkafka::error::{KafkaError, RDKafkaErrorCode};
-use rdkafka::producer::{BaseProducer, BaseRecord};
+use rdkafka::producer::{BaseProducer, BaseRecord, Producer};
 use rdkafka::ClientConfig;
 
 /// Create a kafka topic
@@ -82,22 +83,21 @@ pub async fn producer(broker_addr: &str, datadir: String) {
             .expect("failed to create topic");
 
         let content = std::fs::read(file.path()).unwrap();
-        let msgs: Box<dyn Iterator<Item = (Option<&[u8]>, &[u8])> + Send> =
-            if topic.ends_with("bin") {
-                // binary message data, a file is a message
-                Box::new(std::iter::once((None, content.as_slice())))
-            } else {
-                // text message data, a line is a message
-                Box::new(
-                    content
-                        .split(|&b| b == b'\n')
-                        .filter(|line| !line.is_empty())
-                        .map(|line| match line.iter().position(|&b| b == KEY_DELIMITER) {
-                            Some(pos) => (Some(&line[..pos]), &line[pos + 1..]),
-                            None => (None, line),
-                        }),
-                )
-            };
+        let msgs = if topic.ends_with("bin") {
+            // binary message data, a file is a message
+            Either::Left(std::iter::once((None, content.as_slice())))
+        } else {
+            // text message data, a line is a message
+            Either::Right(
+                content
+                    .split(|&b| b == b'\n')
+                    .filter(|line| !line.is_empty())
+                    .map(|line| match line.iter().position(|&b| b == KEY_DELIMITER) {
+                        Some(pos) => (Some(&line[..pos]), &line[pos + 1..]),
+                        None => (None, line),
+                    }),
+            )
+        };
         for (key, payload) in msgs {
             loop {
                 let ts = SystemTime::now()
diff --git a/src/tests/simulation/src/lib.rs b/src/tests/simulation/src/lib.rs
index 8e0bd81ad6eec..68c1d0446944d 100644
--- a/src/tests/simulation/src/lib.rs
+++ b/src/tests/simulation/src/lib.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![cfg(madsim)]
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
 #![feature(lazy_cell)]
diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs
index c37af587553f3..a3aa4ca056415 100644
--- a/src/tests/simulation/src/main.rs
+++ b/src/tests/simulation/src/main.rs
@@ -17,13 +17,9 @@
 
 use std::path::PathBuf;
 
+use cfg_or_panic::cfg_or_panic;
 use clap::Parser;
 
-#[cfg(not(madsim))]
-fn main() {
-    println!("This binary is only available in simulation.");
-}
-
 /// Deterministic simulation end-to-end test runner.
 ///
 /// ENVS:
@@ -145,8 +141,8 @@ pub struct Args {
     e2e_extended_test: bool,
 }
 
-#[cfg(madsim)]
-#[madsim::main]
+#[tokio::main]
+#[cfg_or_panic(madsim)]
 async fn main() {
     use std::sync::Arc;
 
diff --git a/src/tests/simulation/src/risingwave-scale.toml b/src/tests/simulation/src/risingwave-scale.toml
index cba06c931e3f4..6b476127d8d9f 100644
--- a/src/tests/simulation/src/risingwave-scale.toml
+++ b/src/tests/simulation/src/risingwave-scale.toml
@@ -13,3 +13,4 @@ checkpoint_frequency = 4
 
 [server]
 telemetry_enabled = false
+metrics_level = "Disabled"
diff --git a/src/tests/simulation/src/slt.rs b/src/tests/simulation/src/slt.rs
index c708d16ffc9f9..9df28c1ee24ff 100644
--- a/src/tests/simulation/src/slt.rs
+++ b/src/tests/simulation/src/slt.rs
@@ -24,11 +24,7 @@ use crate::cluster::{Cluster, KillOpts};
 use crate::utils::TimedExt;
 
 fn is_create_table_as(sql: &str) -> bool {
-    let parts: Vec<String> = sql
-        .trim_start()
-        .split_whitespace()
-        .map(|s| s.to_lowercase())
-        .collect();
+    let parts: Vec<String> = sql.split_whitespace().map(|s| s.to_lowercase()).collect();
 
     parts.len() >= 4 && parts[0] == "create" && parts[1] == "table" && parts[3] == "as"
 }
@@ -253,6 +249,9 @@ fn hack_kafka_test(path: &Path) -> tempfile::NamedTempFile {
             .expect("failed to get schema path");
     let proto_full_path = std::fs::canonicalize("src/connector/src/test_data/complex-schema")
         .expect("failed to get schema path");
+    let json_schema_full_path =
+        std::fs::canonicalize("src/connector/src/test_data/complex-schema.json")
+            .expect("failed to get schema path");
     let content = content
         .replace("127.0.0.1:29092", "192.168.11.1:29092")
         .replace(
@@ -266,6 +265,10 @@ fn hack_kafka_test(path: &Path) -> tempfile::NamedTempFile {
         .replace(
             "/risingwave/proto-complex-schema",
             proto_full_path.to_str().unwrap(),
+        )
+        .replace(
+            "/risingwave/json-complex-schema",
+            json_schema_full_path.to_str().unwrap(),
         );
     let file = tempfile::NamedTempFile::new().expect("failed to create temp file");
     std::fs::write(file.path(), content).expect("failed to write file");
diff --git a/src/tests/simulation/src/utils/timed_future.rs b/src/tests/simulation/src/utils/timed_future.rs
index 06b3f7f8f1b4c..bacd1a57aed56 100644
--- a/src/tests/simulation/src/utils/timed_future.rs
+++ b/src/tests/simulation/src/utils/timed_future.rs
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
 
 use pin_project::pin_project;
 
-/// Inspired by https://stackoverflow.com/a/59935743/2990323
+/// Inspired by <https://stackoverflow.com/a/59935743/2990323>
 /// A wrapper around a Future which adds timing data.
 #[pin_project]
 pub struct Timed<Fut, F>
diff --git a/src/tests/simulation/tests/integration_tests/backfill_tests.rs b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
index d42d4b3d500af..813bc3c0b7257 100644
--- a/src/tests/simulation/tests/integration_tests/backfill_tests.rs
+++ b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
@@ -13,16 +13,14 @@
 // limitations under the License.
 
 use anyhow::Result;
-use itertools::{any, Itertools};
+use itertools::Itertools;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 
 const SET_PARALLELISM: &str = "SET STREAMING_PARALLELISM=1;";
 const ROOT_TABLE_CREATE: &str = "create table t1 (_id int, data jsonb);";
 const INSERT_SEED_SQL: &str =
     r#"insert into t1 values (1, '{"orders": {"id": 1, "price": "2.30", "customer_id": 2}}');"#;
-const INSERT_AND_FLUSH_SQL: &str = r#"insert into t1 values (1, '{"orders": {"id": 1, "price": "2.30", "customer_id": 2}}'); FLUSH;"#;
 const INSERT_RECURSE_SQL: &str = "insert into t1 select _id + 1, data from t1;";
-const INSERT_RECURSE_AND_FLUSH_SQL: &str = "insert into t1 select _id + 1, data from t1;";
 const MV1: &str = r#"
 create materialized view mv1 as
 with p1 as (
@@ -47,7 +45,7 @@ select
 from p2;
 "#;
 
-#[madsim::test]
+#[tokio::test]
 async fn test_backfill_with_upstream_and_snapshot_read() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_backfill()).await?;
     let mut session = cluster.start_session();
@@ -70,7 +68,7 @@ async fn test_backfill_with_upstream_and_snapshot_read() -> Result<()> {
     let sessions = (0..3).map(|_| cluster.start_session()).collect_vec();
 
     // Create lots of base table update
-    for mut session in sessions.into_iter() {
+    for mut session in sessions {
         let task = tokio::spawn(async move {
             session.run(INSERT_RECURSE_SQL).await?;
             anyhow::Ok(())
@@ -82,7 +80,7 @@ async fn test_backfill_with_upstream_and_snapshot_read() -> Result<()> {
     let sessions = (0..10).map(|_| cluster.start_session()).collect_vec();
 
     // Create lots of base table update
-    for mut session in sessions.into_iter() {
+    for mut session in sessions {
         let task = tokio::spawn(async move {
             for _ in 0..10 {
                 session.run("FLUSH;").await?;
diff --git a/src/tests/simulation/tests/integration_tests/batch/mod.rs b/src/tests/simulation/tests/integration_tests/batch/mod.rs
index 52bce7212c41b..cebf43fa36767 100644
--- a/src/tests/simulation/tests/integration_tests/batch/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/batch/mod.rs
@@ -12,12 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg(madsim)]
+
 use std::io::Write;
 
-use anyhow::Result;
 use clap::Parser;
 use itertools::Itertools;
-use madsim::runtime::Handle;
 use risingwave_simulation::cluster::{Cluster, ConfigPath, Configuration, Session};
 use tokio::time::Duration;
 
@@ -62,6 +62,7 @@ checkpoint_frequency = 1
 
 [server]
 telemetry_enabled = false
+metrics_level = \"Disabled\"
         "
             .as_bytes(),
         )
@@ -81,7 +82,7 @@ telemetry_enabled = false
     }
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_serving_cluster_availability() {
     let config = cluster_config_no_compute_nodes();
     let mut cluster = Cluster::start(config).await.unwrap();
diff --git a/src/tests/simulation/tests/integration_tests/main.rs b/src/tests/simulation/tests/integration_tests/main.rs
index 095f9709c2791..d7c79a0ca37f3 100644
--- a/src/tests/simulation/tests/integration_tests/main.rs
+++ b/src/tests/simulation/tests/integration_tests/main.rs
@@ -18,7 +18,6 @@
 //! for the rationale behind this approach.
 
 #![feature(stmt_expr_attributes)]
-#![cfg(madsim)]
 #![feature(lazy_cell)]
 #![feature(extract_if)]
 
diff --git a/src/tests/simulation/tests/integration_tests/recovery/backfill.rs b/src/tests/simulation/tests/integration_tests/recovery/backfill.rs
index a83c217cf151d..e907528826864 100644
--- a/src/tests/simulation/tests/integration_tests/recovery/backfill.rs
+++ b/src/tests/simulation/tests/integration_tests/recovery/backfill.rs
@@ -17,10 +17,10 @@ use std::time::Duration;
 
 use anyhow::Result;
 use itertools::Itertools;
-use madsim::time::sleep;
 use risingwave_simulation::cluster::{Cluster, Configuration, Session};
 use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains};
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 const ROOT_TABLE_CREATE: &str = "create table t1 (v1 int);";
 const ROOT_TABLE_DROP: &str = "drop table t1;";
@@ -46,7 +46,7 @@ async fn test_no_backfill_state(session: &mut Session) -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_snapshot_mv() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -98,7 +98,7 @@ async fn test_snapshot_mv() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_backfill_mv() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -151,7 +151,7 @@ async fn test_backfill_mv() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_index_backfill() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
diff --git a/src/tests/simulation/tests/integration_tests/recovery/mod.rs b/src/tests/simulation/tests/integration_tests/recovery/mod.rs
index 3b45a0d63853e..565487e8d7dbd 100644
--- a/src/tests/simulation/tests/integration_tests/recovery/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/recovery/mod.rs
@@ -14,3 +14,4 @@
 
 mod backfill;
 mod nexmark_recovery;
+mod pause_on_bootstrap;
diff --git a/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs b/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs
index 1720c2edc21e7..edd419adabf66 100644
--- a/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs
+++ b/src/tests/simulation/tests/integration_tests/recovery/nexmark_recovery.rs
@@ -15,10 +15,10 @@
 use std::time::Duration;
 
 use anyhow::Result;
-use madsim::time::{sleep, Instant};
 use risingwave_simulation::cluster::{Configuration, KillOpts};
-use risingwave_simulation::nexmark::{self, NexmarkCluster, THROUGHPUT};
+use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT};
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 /// Setup a nexmark stream, inject failures, and verify results.
 async fn nexmark_recovery_common(create: &str, select: &str, drop: &str) -> Result<()> {
@@ -54,7 +54,7 @@ async fn nexmark_recovery_common(create: &str, select: &str, drop: &str) -> Resu
 macro_rules! test {
     ($query:ident) => {
         paste::paste! {
-            #[madsim::test]
+            #[tokio::test]
             async fn [< nexmark_recovery_ $query >]() -> Result<()> {
                 use risingwave_simulation::nexmark::queries::$query::*;
                 nexmark_recovery_common(CREATE, SELECT, DROP)
diff --git a/src/tests/simulation/tests/integration_tests/recovery/pause_on_bootstrap.rs b/src/tests/simulation/tests/integration_tests/recovery/pause_on_bootstrap.rs
new file mode 100644
index 0000000000000..d0288e6931e88
--- /dev/null
+++ b/src/tests/simulation/tests/integration_tests/recovery/pause_on_bootstrap.rs
@@ -0,0 +1,125 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use anyhow::Result;
+use risingwave_simulation::cluster::Configuration;
+use risingwave_simulation::nexmark::NexmarkCluster;
+use risingwave_simulation::utils::AssertResult;
+use tokio::time::{sleep, timeout};
+
+const CREATE_TABLE: &str = "CREATE TABLE t (v int)";
+const INSERT_INTO_TABLE: &str = "INSERT INTO t VALUES (1)";
+const SELECT_COUNT_TABLE: &str = "SELECT COUNT(*) FROM t";
+
+const CREATE: &str = "CREATE MATERIALIZED VIEW count_bid as SELECT COUNT(*) FROM bid";
+const SELECT: &str = "SELECT * FROM count_bid";
+
+const CREATE_2: &str = "CREATE MATERIALIZED VIEW count_auction as SELECT COUNT(*) FROM auction";
+const SELECT_2: &str = "SELECT * FROM count_auction";
+
+const SET_PARAMETER: &str = "ALTER SYSTEM SET pause_on_next_bootstrap TO true";
+
+enum ResumeBy {
+    Risectl,
+    Restart,
+}
+
+async fn test_impl(resume_by: ResumeBy) -> Result<()> {
+    let mut cluster = NexmarkCluster::new(
+        Configuration {
+            meta_nodes: 1,
+            ..Configuration::for_scale()
+        },
+        6,
+        None,
+        false,
+    )
+    .await?;
+
+    cluster.run(SET_PARAMETER).await?;
+    cluster.run(CREATE).await?;
+    cluster.run(CREATE_TABLE).await?;
+
+    // Run for a while.
+    sleep(Duration::from_secs(10)).await;
+
+    // Kill the meta node and wait for the service to recover.
+    cluster.kill_nodes(["meta-1"], 0).await;
+    sleep(Duration::from_secs(10)).await;
+
+    // The source should be paused.
+    let count = cluster.run(SELECT).await?;
+    sleep(Duration::from_secs(10)).await;
+    cluster.run(SELECT).await?.assert_result_eq(&count);
+
+    // Scaling will trigger a pair of `Pause` and `Resume`. However, this should not affect the
+    // "manual" pause.
+    let random_fragment_id = cluster.locate_random_fragment().await?;
+    cluster
+        .reschedule(random_fragment_id.random_reschedule())
+        .await?;
+    sleep(Duration::from_secs(10)).await;
+    cluster.run(SELECT).await?.assert_result_eq(&count);
+
+    // New streaming jobs should also start from paused.
+    cluster.run(CREATE_2).await?;
+    sleep(Duration::from_secs(10)).await;
+    cluster.run(SELECT_2).await?.assert_result_eq("0"); // even there's no data from source, the
+                                                        // result will be 0 instead of empty or NULL
+
+    // DML on tables should be blocked.
+    let result = timeout(Duration::from_secs(10), cluster.run(INSERT_INTO_TABLE)).await;
+    assert!(result.is_err());
+    cluster.run(SELECT_COUNT_TABLE).await?.assert_result_eq("0");
+
+    match resume_by {
+        ResumeBy::Risectl => cluster.resume().await?,
+        ResumeBy::Restart => cluster.kill_nodes(["meta-1"], 0).await,
+    }
+    sleep(Duration::from_secs(10)).await;
+
+    // The source should be resumed.
+    let new_count = cluster.run(SELECT).await?;
+    assert_ne!(count, new_count);
+
+    // DML on tables should be allowed. However, we're uncertain whether the previous blocked DML is
+    // executed or not. So we just check the count difference.
+    {
+        let mut session = cluster.start_session();
+
+        session.run("FLUSH").await?;
+        let count: i64 = session.run(SELECT_COUNT_TABLE).await?.parse().unwrap();
+
+        session.run(INSERT_INTO_TABLE).await?;
+        session.run("FLUSH").await?;
+        session
+            .run(SELECT_COUNT_TABLE)
+            .await?
+            .assert_result_eq(format!("{}", count + 1));
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_pause_on_bootstrap_resume_by_risectl() -> Result<()> {
+    test_impl(ResumeBy::Risectl).await
+}
+
+#[tokio::test]
+async fn test_pause_on_bootstrap_resume_by_restart() -> Result<()> {
+    test_impl(ResumeBy::Restart).await
+}
diff --git a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs
index 968762570ec79..c05e52c927424 100644
--- a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs
@@ -16,10 +16,10 @@ use std::time::Duration;
 
 use anyhow::Result;
 use itertools::Itertools;
-use madsim::time::sleep;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains};
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 const ROOT_TABLE_CREATE: &str = "create table t1 (v1 int);";
 const MV1: &str = "create materialized view m1 as select * from t1 where v1 > 5;";
@@ -28,7 +28,7 @@ const MV3: &str = "create materialized view m3 as select * from m2 where v1 < 15
 const MV4: &str = "create materialized view m4 as select m1.v1 as m1v, m3.v1 as m3v from m1 join m3 on m1.v1 = m3.v1;";
 const MV5: &str = "create materialized view m5 as select * from m4;";
 
-#[madsim::test]
+#[tokio::test]
 async fn test_simple_cascade_materialized_view() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -113,7 +113,7 @@ async fn test_simple_cascade_materialized_view() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_diamond_cascade_materialized_view() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
diff --git a/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs b/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs
index a9028836a8c54..e8e1af703c1c2 100644
--- a/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs
@@ -16,14 +16,14 @@ use std::collections::HashSet;
 use std::time::Duration;
 
 use anyhow::Result;
-use madsim::time::sleep;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 const SELECT: &str = "select * from mv1 order by v1;";
 
-#[madsim::test]
+#[tokio::test]
 async fn test_dynamic_filter() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
diff --git a/src/tests/simulation/tests/integration_tests/scale/mod.rs b/src/tests/simulation/tests/integration_tests/scale/mod.rs
index 2fb163ffea443..5270247bf3997 100644
--- a/src/tests/simulation/tests/integration_tests/scale/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/mod.rs
@@ -21,5 +21,6 @@ mod no_shuffle;
 mod plan;
 mod schedulability;
 mod singleton_migration;
+mod sink;
 mod streaming_parallelism;
 mod table;
diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs
index a06fb25fe7a18..6ed30a7452dcc 100644
--- a/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_chaos.rs
@@ -17,11 +17,11 @@ use std::time::Duration;
 use anyhow::Result;
 use futures::future::BoxFuture;
 use itertools::Itertools;
-use madsim::time::sleep;
 use risingwave_simulation::cluster::Configuration;
 use risingwave_simulation::ctl_ext::Fragment;
 use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT};
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 /// Common code for Nexmark chaos tests.
 ///
@@ -117,7 +117,7 @@ macro_rules! test {
     };
     ($query:ident, $after_scale_duration:expr) => {
         paste::paste! {
-            #[madsim::test]
+            #[tokio::test]
             async fn [< nexmark_chaos_ $query _single >]() -> Result<()> {
                 use risingwave_simulation::nexmark::queries::$query::*;
                 nexmark_chaos_common(
@@ -132,7 +132,7 @@ macro_rules! test {
                 .await
             }
 
-            #[madsim::test]
+            #[tokio::test]
             async fn [< nexmark_chaos_ $query _multiple >]() -> Result<()> {
                 use risingwave_simulation::nexmark::queries::$query::*;
                 nexmark_chaos_common(
diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs
index ed3f3d4ad580f..41a90fcf5d0d5 100644
--- a/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs
@@ -15,7 +15,6 @@
 use std::time::Duration;
 
 use anyhow::Result;
-use madsim::time::sleep;
 use risingwave_simulation::cluster::Configuration;
 use risingwave_simulation::ctl_ext::predicate::{
     identity_contains, upstream_fragment_count, BoxedPredicate,
@@ -23,6 +22,7 @@ use risingwave_simulation::ctl_ext::predicate::{
 use risingwave_simulation::nexmark::queries::q4::*;
 use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT};
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 #[cfg(target_os = "linux")]
 const RESULT: &str = r#"
@@ -55,7 +55,7 @@ async fn wait_initial_data(cluster: &mut NexmarkCluster) -> Result<String> {
         .await
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_q4_ref() -> Result<()> {
     let mut cluster = init().await?;
 
@@ -93,7 +93,7 @@ async fn nexmark_q4_common(predicates: impl IntoIterator<Item = BoxedPredicate>)
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_q4_materialize_agg() -> Result<()> {
     nexmark_q4_common([
         identity_contains("materialize"),
@@ -102,12 +102,12 @@ async fn nexmark_q4_materialize_agg() -> Result<()> {
     .await
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_q4_source() -> Result<()> {
     nexmark_q4_common([identity_contains("source: bid")]).await
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_q4_agg_join() -> Result<()> {
     nexmark_q4_common([
         identity_contains("hashagg"),
@@ -117,7 +117,7 @@ async fn nexmark_q4_agg_join() -> Result<()> {
     .await
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_q4_cascade() -> Result<()> {
     let mut cluster = init().await?;
 
@@ -165,7 +165,7 @@ async fn nexmark_q4_cascade() -> Result<()> {
 }
 
 // https://github.com/risingwavelabs/risingwave/issues/5567
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_q4_materialize_agg_cache_invalidation() -> Result<()> {
     let mut cluster = init().await?;
 
diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs
index d1d6d1814f38b..864195d0efd7b 100644
--- a/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_source.rs
@@ -15,17 +15,17 @@
 use std::time::Duration;
 
 use anyhow::Result;
-use madsim::time::sleep;
 use risingwave_simulation::cluster::Configuration;
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 use risingwave_simulation::nexmark::{NexmarkCluster, THROUGHPUT};
+use tokio::time::sleep;
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_source() -> Result<()> {
     nexmark_source_inner(false).await
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn nexmark_source_with_watermark() -> Result<()> {
     nexmark_source_inner(true).await
 }
diff --git a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs
index 3a8402d5a83ae..0fde294d08e7f 100644
--- a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs
@@ -18,7 +18,7 @@ use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 use risingwave_simulation::utils::AssertResult;
 
-#[madsim::test]
+#[tokio::test]
 async fn test_delta_join() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -120,7 +120,7 @@ async fn test_delta_join() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_share_multiple_no_shuffle_upstream() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -140,7 +140,7 @@ async fn test_share_multiple_no_shuffle_upstream() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_resolve_no_shuffle_upstream() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
diff --git a/src/tests/simulation/tests/integration_tests/scale/plan.rs b/src/tests/simulation/tests/integration_tests/scale/plan.rs
index 0c2a2d2d396e0..c7244dc826b42 100644
--- a/src/tests/simulation/tests/integration_tests/scale/plan.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/plan.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::default::Default;
 
 use anyhow::Result;
 use itertools::Itertools;
@@ -26,7 +27,7 @@ use risingwave_pb::meta::PbReschedule;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains};
 
-#[madsim::test]
+#[tokio::test]
 async fn test_resize_normal() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -65,7 +66,7 @@ async fn test_resize_normal() -> Result<()> {
                 WorkerChanges {
                     include_worker_ids: vec![],
                     exclude_worker_ids: removed_workers,
-                    target_parallelism: None,
+                    ..Default::default()
                 },
             )]),
         }))
@@ -92,7 +93,7 @@ async fn test_resize_normal() -> Result<()> {
 
     Ok(())
 }
-#[madsim::test]
+#[tokio::test]
 async fn test_resize_single() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -135,7 +136,7 @@ async fn test_resize_single() -> Result<()> {
                 .parallel_units
                 .iter()
                 .map(|parallel_unit| parallel_unit.id)
-                .contains(&used_parallel_unit_id)
+                .contains(used_parallel_unit_id)
         })
         .collect_vec();
 
@@ -148,7 +149,7 @@ async fn test_resize_single() -> Result<()> {
                 WorkerChanges {
                     include_worker_ids: vec![],
                     exclude_worker_ids: vec![prev_worker.id],
-                    target_parallelism: None,
+                    ..Default::default()
                 },
             )]),
         }))
@@ -175,7 +176,7 @@ async fn test_resize_single() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_resize_single_failed() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -223,7 +224,7 @@ async fn test_resize_single_failed() -> Result<()> {
                     WorkerChanges {
                         include_worker_ids: vec![],
                         exclude_worker_ids: vec![worker_a.id],
-                        target_parallelism: None,
+                        ..Default::default()
                     },
                 ),
                 (
@@ -231,7 +232,7 @@ async fn test_resize_single_failed() -> Result<()> {
                     WorkerChanges {
                         include_worker_ids: vec![],
                         exclude_worker_ids: vec![worker_b.id],
-                        target_parallelism: None,
+                        ..Default::default()
                     },
                 ),
             ]),
@@ -242,7 +243,7 @@ async fn test_resize_single_failed() -> Result<()> {
 
     Ok(())
 }
-#[madsim::test]
+#[tokio::test]
 async fn test_resize_no_shuffle() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -302,7 +303,7 @@ join mv5 on mv1.v = mv5.v;",
                 WorkerChanges {
                     include_worker_ids: vec![],
                     exclude_worker_ids: removed_worker_ids,
-                    target_parallelism: None,
+                    ..Default::default()
                 },
             )]),
         }))
diff --git a/src/tests/simulation/tests/integration_tests/scale/schedulability.rs b/src/tests/simulation/tests/integration_tests/scale/schedulability.rs
index ed5c5ec468771..5ad82b5326732 100644
--- a/src/tests/simulation/tests/integration_tests/scale/schedulability.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/schedulability.rs
@@ -19,7 +19,7 @@ use risingwave_common::hash::ParallelUnitId;
 use risingwave_pb::common::{WorkerNode, WorkerType};
 use risingwave_simulation::cluster::{Cluster, Configuration};
 
-#[madsim::test]
+#[tokio::test]
 async fn test_cordon_normal() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
@@ -78,7 +78,7 @@ async fn test_cordon_normal() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_cordon_no_shuffle_failed() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
diff --git a/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs b/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs
index 51c4cb17a6387..6abd980bea3bd 100644
--- a/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs
@@ -16,18 +16,18 @@ use std::time::Duration;
 
 use anyhow::Result;
 use itertools::Itertools;
-use madsim::time::sleep;
 use rand::prelude::SliceRandom;
 use rand::thread_rng;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 use risingwave_simulation::utils::AssertResult;
+use tokio::time::sleep;
 
 const ROOT_TABLE_CREATE: &str = "create table t (v1 int);";
 const ROOT_MV: &str = "create materialized view m1 as select count(*) as c1 from t;";
 const CASCADE_MV: &str = "create materialized view m2 as select * from m1;";
 
-#[madsim::test]
+#[tokio::test]
 async fn test_singleton_migration() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let mut session = cluster.start_session();
diff --git a/src/tests/simulation/tests/integration_tests/scale/sink.rs b/src/tests/simulation/tests/integration_tests/scale/sink.rs
index 414acd0ad05e6..99f150e85b8a1 100644
--- a/src/tests/simulation/tests/integration_tests/scale/sink.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/sink.rs
@@ -16,15 +16,16 @@ use std::collections::HashMap;
 use std::time::Duration;
 
 use anyhow::Result;
-use madsim::export::futures::StreamExt;
-use madsim::rand::prelude::SliceRandom;
-use madsim::rand::thread_rng;
-use madsim::time;
-use rdkafka::consumer::{MessageStream, StreamConsumer};
+use futures::StreamExt;
+use rand::prelude::SliceRandom;
+use rand::thread_rng;
+use rdkafka::consumer::{Consumer, StreamConsumer};
+use rdkafka::error::KafkaResult;
 use rdkafka::message::BorrowedMessage;
 use rdkafka::{ClientConfig, Message, TopicPartitionList};
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains};
+use tokio::time;
 
 const ROOT_TABLE_CREATE: &str = "create table t (v1 int) append only;";
 const APPEND_ONLY_SINK_CREATE: &str = "create sink s1 from t with (connector='kafka', properties.bootstrap.server='192.168.11.1:29092', topic='t_sink_append_only', type='append-only');";
@@ -70,7 +71,8 @@ pub struct Before {
     pub count: i64,
 }
 
-#[madsim::test]
+#[tokio::test]
+#[ignore] // https://github.com/risingwavelabs/risingwave/issues/12003
 async fn test_sink_append_only() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
 
@@ -122,7 +124,8 @@ async fn test_sink_append_only() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
+#[ignore] // https://github.com/risingwavelabs/risingwave/issues/12003
 async fn test_sink_debezium() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
 
@@ -201,7 +204,7 @@ async fn test_sink_debezium() -> Result<()> {
     Ok(())
 }
 
-fn check_payload(msg: &BorrowedMessage, payload: &[u8], i: i64) {
+fn check_payload(msg: &BorrowedMessage<'_>, payload: &[u8], i: i64) {
     match msg.topic() {
         APPEND_ONLY_TOPIC => {
             let data: AppendOnlyPayload = serde_json::from_slice(payload).unwrap();
@@ -228,7 +231,7 @@ fn check_payload(msg: &BorrowedMessage, payload: &[u8], i: i64) {
 
 async fn check_kafka_after_insert(
     cluster: &mut Cluster,
-    stream: &mut MessageStream<'_>,
+    stream: &mut (impl futures::Stream<Item = KafkaResult<BorrowedMessage<'_>>> + Unpin),
     input: &[i64],
 ) -> Result<()> {
     for i in input {
diff --git a/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs b/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs
index 8828b7380616d..1e1260e7b0bd2 100644
--- a/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/streaming_parallelism.rs
@@ -16,7 +16,7 @@ use anyhow::Result;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 
-#[madsim::test]
+#[tokio::test]
 async fn test_streaming_parallelism_default() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores;
@@ -28,7 +28,7 @@ async fn test_streaming_parallelism_default() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_streaming_parallelism_set_some() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores;
@@ -48,7 +48,7 @@ async fn test_streaming_parallelism_set_some() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_streaming_parallelism_set_zero() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores;
@@ -64,7 +64,7 @@ async fn test_streaming_parallelism_set_zero() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_streaming_parallelism_mv_on_mv() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores;
@@ -101,7 +101,7 @@ async fn test_streaming_parallelism_mv_on_mv() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_streaming_parallelism_index() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     let default_parallelism = cluster.config().compute_nodes * cluster.config().compute_node_cores;
diff --git a/src/tests/simulation/tests/integration_tests/scale/table.rs b/src/tests/simulation/tests/integration_tests/scale/table.rs
index 4ab4c2ce9ec25..8044818b43d0a 100644
--- a/src/tests/simulation/tests/integration_tests/scale/table.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/table.rs
@@ -12,17 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::iter::{repeat, repeat_with};
-use std::time::Duration;
+use std::iter::repeat_with;
 
 use anyhow::Result;
 use itertools::Itertools;
-use madsim::time::sleep;
-use rand::prelude::SliceRandom;
-use rand::thread_rng;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
-use risingwave_simulation::utils::AssertResult;
 
 const ROOT_TABLE_CREATE: &str = "create table t (v1 int);";
 const MV1: &str = "create materialized view m1 as select * from t;";
@@ -41,7 +36,7 @@ macro_rules! insert_and_flush {
     }};
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_table() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     cluster.run(ROOT_TABLE_CREATE).await?;
@@ -65,7 +60,7 @@ async fn test_table() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_mv_on_scaled_table() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::for_scale()).await?;
     cluster.run(ROOT_TABLE_CREATE).await?;
diff --git a/src/tests/simulation/tests/integration_tests/sink/basic.rs b/src/tests/simulation/tests/integration_tests/sink/basic.rs
index 6747b8bbac4dd..a12bc3643b542 100644
--- a/src/tests/simulation/tests/integration_tests/sink/basic.rs
+++ b/src/tests/simulation/tests/integration_tests/sink/basic.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::future::pending;
 use std::io::Write;
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering::Relaxed;
@@ -22,7 +21,6 @@ use std::time::Duration;
 use anyhow::Result;
 use async_trait::async_trait;
 use itertools::Itertools;
-use madsim::time::{sleep, timeout};
 use rand::prelude::SliceRandom;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
@@ -30,6 +28,7 @@ use risingwave_connector::sink::boxed::{BoxCoordinator, BoxWriter};
 use risingwave_connector::sink::test_sink::registry_build_sink;
 use risingwave_connector::sink::{Sink, SinkWriter, SinkWriterParam};
 use risingwave_simulation::cluster::{Cluster, ConfigPath, Configuration};
+use tokio::time::sleep;
 
 struct TestWriter {
     row_counter: Arc<AtomicUsize>,
@@ -58,17 +57,6 @@ impl SinkWriter for TestWriter {
         sleep(Duration::from_millis(100)).await;
         Ok(())
     }
-
-    async fn abort(&mut self) -> risingwave_connector::sink::Result<()> {
-        Ok(())
-    }
-
-    async fn update_vnode_bitmap(
-        &mut self,
-        _vnode_bitmap: Bitmap,
-    ) -> risingwave_connector::sink::Result<()> {
-        Ok(())
-    }
 }
 
 impl Drop for TestWriter {
@@ -106,7 +94,7 @@ impl Sink for TestSink {
     }
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_sink_basic() -> Result<()> {
     let config_path = {
         let mut file = tempfile::NamedTempFile::new().expect("failed to create temp config file");
@@ -133,7 +121,7 @@ async fn test_sink_basic() -> Result<()> {
     let _sink_guard = registry_build_sink({
         let row_counter = row_counter.clone();
         let parallelism_counter = parallelism_counter.clone();
-        move |param| {
+        move |_param| {
             Ok(Box::new(TestSink {
                 row_counter: row_counter.clone(),
                 parallelism_counter: parallelism_counter.clone(),
@@ -178,7 +166,7 @@ async fn test_sink_basic() -> Result<()> {
     Ok(())
 }
 
-#[madsim::test]
+#[tokio::test]
 async fn test_sink_decouple_basic() -> Result<()> {
     let config_path = {
         let mut file = tempfile::NamedTempFile::new().expect("failed to create temp config file");
@@ -205,7 +193,7 @@ async fn test_sink_decouple_basic() -> Result<()> {
     let _sink_guard = registry_build_sink({
         let row_counter = row_counter.clone();
         let parallelism_counter = parallelism_counter.clone();
-        move |param| {
+        move |_param| {
             Ok(Box::new(TestSink {
                 row_counter: row_counter.clone(),
                 parallelism_counter: parallelism_counter.clone(),
diff --git a/src/tests/simulation/tests/integration_tests/sink/mod.rs b/src/tests/simulation/tests/integration_tests/sink/mod.rs
index 6db319067a068..71a65bf062d9c 100644
--- a/src/tests/simulation/tests/integration_tests/sink/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/sink/mod.rs
@@ -12,4 +12,5 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(madsim)]
 mod basic;
diff --git a/src/tests/simulation/tests/integration_tests/storage.rs b/src/tests/simulation/tests/integration_tests/storage.rs
index 193d00395d16d..e300a8194e453 100644
--- a/src/tests/simulation/tests/integration_tests/storage.rs
+++ b/src/tests/simulation/tests/integration_tests/storage.rs
@@ -16,10 +16,9 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
 use anyhow::Result;
-use madsim::rand::thread_rng;
 use rand::distributions::{Alphanumeric, DistString};
 use rand::rngs::SmallRng;
-use rand::{RngCore, SeedableRng};
+use rand::{thread_rng, RngCore, SeedableRng};
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::utils::AssertResult;
 
@@ -27,7 +26,7 @@ const NUM_ROWS: usize = 500;
 const NUM_OVERWRITES: usize = 5000;
 const MAX_STRING_LEN: usize = 150;
 
-#[madsim::test]
+#[tokio::test]
 async fn test_storage_with_random_writes() -> Result<()> {
     // TODO: Use backfill configuration for now
     let mut cluster = Cluster::start(Configuration::for_backfill()).await?;
diff --git a/src/tests/sqlsmith/src/runner.rs b/src/tests/sqlsmith/src/runner.rs
index 2e796aefdc188..5efc793cdd95c 100644
--- a/src/tests/sqlsmith/src/runner.rs
+++ b/src/tests/sqlsmith/src/runner.rs
@@ -538,7 +538,7 @@ async fn run_query_inner(
 /// Create the tables defined in testdata, along with some mviews.
 /// Just test number of rows for now.
 /// TODO(kwannoel): Test row contents as well. That requires us to run a batch query
-/// with select * ORDER BY <all columns>.
+/// with `select * ORDER BY <all columns>`.
 async fn diff_stream_and_batch(
     rng: &mut impl Rng,
     mvs_and_base_tables: Vec<Table>,
diff --git a/src/tests/sqlsmith/src/sql_gen/scalar.rs b/src/tests/sqlsmith/src/sql_gen/scalar.rs
index 63762e8177f36..b10ab91fb123d 100644
--- a/src/tests/sqlsmith/src/sql_gen/scalar.rs
+++ b/src/tests/sqlsmith/src/sql_gen/scalar.rs
@@ -135,7 +135,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
         }
     }
 
-    /// Generates a list of [`n`] simple scalar values of a specific [`type`].
+    /// Generates a list of `n` simple scalar values of a specific `type`.
     fn gen_simple_scalar_list(&mut self, ty: &DataType, n: usize) -> Vec<Expr> {
         (0..n).map(|_| self.gen_simple_scalar(ty)).collect()
     }
diff --git a/src/udf/Cargo.toml b/src/udf/Cargo.toml
index 34e3dbc3e922e..b82fad73cc60c 100644
--- a/src/udf/Cargo.toml
+++ b/src/udf/Cargo.toml
@@ -11,10 +11,10 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
-arrow-array = "45"
-arrow-flight = "45"
-arrow-schema = "45"
-arrow-select = "45"
+arrow-array = { workspace = true }
+arrow-flight = { workspace = true }
+arrow-schema = { workspace = true }
+arrow-select = { workspace = true }
 futures-util = "0.3.28"
 static_assertions = "1"
 thiserror = "1"
diff --git a/src/udf/python/.gitignore b/src/udf/python/.gitignore
new file mode 100644
index 0000000000000..75b18b1dc1919
--- /dev/null
+++ b/src/udf/python/.gitignore
@@ -0,0 +1,2 @@
+/dist
+/risingwave.egg-info
diff --git a/src/udf/python/README.md b/src/udf/python/README.md
index 9386313caf3ac..d1655be05350b 100644
--- a/src/udf/python/README.md
+++ b/src/udf/python/README.md
@@ -1,6 +1,8 @@
-# RisingWave Python API
+# RisingWave Python UDF SDK
 
-This library provides a Python API for creating user-defined functions (UDF) in RisingWave.
+This library provides a Python SDK for creating user-defined functions (UDF) in [RisingWave](https://www.risingwave.com/).
+
+For a detailed guide on how to use Python UDF in RisingWave, please refer to [this doc](https://docs.risingwave.com/docs/current/udf-python/).
 
 ## Introduction
 
diff --git a/src/udf/python/publish.md b/src/udf/python/publish.md
new file mode 100644
index 0000000000000..0bc22d713906f
--- /dev/null
+++ b/src/udf/python/publish.md
@@ -0,0 +1,19 @@
+# How to publish this library
+
+Install the build tool:
+
+```sh
+pip3 install build
+```
+
+Build the library:
+
+```sh
+python3 -m build
+```
+
+Upload the library to PyPI:
+
+```sh
+twine upload dist/*
+```
diff --git a/src/udf/python/pyproject.toml b/src/udf/python/pyproject.toml
new file mode 100644
index 0000000000000..d3093156eff56
--- /dev/null
+++ b/src/udf/python/pyproject.toml
@@ -0,0 +1,20 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "risingwave"
+version = "0.0.10"
+authors = [{ name = "RisingWave Labs" }]
+description = "RisingWave Python API"
+readme = "README.md"
+license = { text = "Apache Software License" }
+classifiers = [
+    "Programming Language :: Python",
+    "License :: OSI Approved :: Apache Software License",
+]
+requires-python = ">=3.8"
+dependencies = ["pyarrow"]
+
+[project.optional-dependencies]
+test = ["pytest"]
diff --git a/src/udf/python/setup.py b/src/udf/python/setup.py
deleted file mode 100644
index bb05b4cc3ef5d..0000000000000
--- a/src/udf/python/setup.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from setuptools import find_packages, setup
-
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
-setup(
-    name="risingwave",
-    version="0.0.9",
-    author="RisingWave Labs",
-    description="RisingWave Python API",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/risingwavelabs/risingwave",
-    packages=find_packages(),
-    classifiers=[
-        "Programming Language :: Python",
-        "License :: OSI Approved :: Apache Software License",
-    ],
-    python_requires=">=3.8",
-    install_requires=["pyarrow"],
-    extras_require={"test": ["pytest"]},
-)
diff --git a/src/utils/pgwire/Cargo.toml b/src/utils/pgwire/Cargo.toml
index b11bdd241eaf0..4ca6b556bf281 100644
--- a/src/utils/pgwire/Cargo.toml
+++ b/src/utils/pgwire/Cargo.toml
@@ -20,7 +20,7 @@ byteorder = "1.4"
 bytes = "1"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 itertools = "0.11"
-openssl = "0.10.56"
+openssl = "0.10.57"
 panic-message = "0.3"
 risingwave_common = { workspace = true }
 risingwave_sqlparser = { workspace = true }
diff --git a/src/utils/pgwire/src/error_or_notice.rs b/src/utils/pgwire/src/error_or_notice.rs
index cc527e2fc614b..a8bf5175c4329 100644
--- a/src/utils/pgwire/src/error_or_notice.rs
+++ b/src/utils/pgwire/src/error_or_notice.rs
@@ -67,7 +67,7 @@ impl Severity {
     }
 }
 
-/// Code: the SQLSTATE code for the error (see https://www.postgresql.org/docs/current/errcodes-appendix.html).
+/// Code: the SQLSTATE code for the error (see <https://www.postgresql.org/docs/current/errcodes-appendix.html>).
 /// Not localizable. Always present.
 #[derive(PartialEq, Eq, Clone, Debug)]
 #[allow(clippy::upper_case_acronyms)]
diff --git a/src/utils/pgwire/src/pg_protocol.rs b/src/utils/pgwire/src/pg_protocol.rs
index 582c0a6603cd1..ff705025a0d64 100644
--- a/src/utils/pgwire/src/pg_protocol.rs
+++ b/src/utils/pgwire/src/pg_protocol.rs
@@ -871,7 +871,7 @@ pub struct PgStream<S> {
 ///  * `integer_datetimes`
 ///  * `standard_conforming_string`
 ///
-/// See: https://www.postgresql.org/docs/9.2/static/protocol-flow.html#PROTOCOL-ASYNC.
+/// See: <https://www.postgresql.org/docs/9.2/static/protocol-flow.html#PROTOCOL-ASYNC>.
 #[derive(Debug, Default, Clone)]
 pub struct ParameterStatus {
     pub application_name: Option<String>,
diff --git a/src/utils/pgwire/src/pg_server.rs b/src/utils/pgwire/src/pg_server.rs
index 62dc93796106b..ba52215e4d34a 100644
--- a/src/utils/pgwire/src/pg_server.rs
+++ b/src/utils/pgwire/src/pg_server.rs
@@ -56,7 +56,7 @@ pub trait Session: Send + Sync {
     type Portal: Send + Clone + std::fmt::Display + 'static;
 
     /// The str sql can not use the unparse from AST: There is some problem when dealing with create
-    /// view, see  https://github.com/risingwavelabs/risingwave/issues/6801.
+    /// view, see <https://github.com/risingwavelabs/risingwave/issues/6801>.
     fn run_one_query(
         self: Arc<Self>,
         sql: Statement,
diff --git a/src/utils/runtime/Cargo.toml b/src/utils/runtime/Cargo.toml
index adff273ed35f7..84c194341e141 100644
--- a/src/utils/runtime/Cargo.toml
+++ b/src/utils/runtime/Cargo.toml
@@ -43,7 +43,7 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
     "fs"
 ] }
 tracing = "0.1"
-tracing-opentelemetry = "0.20"
+tracing-opentelemetry = "0.21"
 tracing-subscriber = { version = "0.3", features = ["fmt", "parking_lot", "std", "time", "local-time", "json"] }
 
 [target.'cfg(not(madsim))'.dependencies]
diff --git a/src/utils/runtime/src/logger.rs b/src/utils/runtime/src/logger.rs
index 6ccacc16b50d1..4a4b77936b800 100644
--- a/src/utils/runtime/src/logger.rs
+++ b/src/utils/runtime/src/logger.rs
@@ -48,8 +48,6 @@ fn configure_risingwave_targets_fmt(targets: filter::Targets) -> filter::Targets
         .with_target("foyer_memory", Level::WARN)
         .with_target("foyer_storage", Level::WARN)
         // disable events that are too verbose
-        // if you want to enable any of them, find the target name and set it to `TRACE`
-        // .with_target("events::stream::mview::scan", Level::TRACE)
         .with_target("events", Level::ERROR)
 }
 
diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml
index 7a811b4406566..99d5f1674f23a 100644
--- a/src/workspace-hack/Cargo.toml
+++ b/src/workspace-hack/Cargo.toml
@@ -23,6 +23,7 @@ anyhow = { version = "1", features = ["backtrace"] }
 aws-credential-types = { version = "0.55", default-features = false, features = ["hardcoded-credentials"] }
 aws-sdk-s3 = { version = "0.28", features = ["native-tls"] }
 aws-smithy-client = { version = "0.55", default-features = false, features = ["native-tls", "rustls"] }
+byteorder = { version = "1", features = ["i128"] }
 bytes = { version = "1", features = ["serde"] }
 chrono = { version = "0.4", features = ["alloc", "serde"] }
 clap = { version = "4", features = ["cargo", "derive", "env"] }
@@ -30,9 +31,12 @@ clap_builder = { version = "4", default-features = false, features = ["cargo", "
 combine = { version = "4" }
 crossbeam-epoch = { version = "0.9" }
 crossbeam-utils = { version = "0.8" }
+deranged = { version = "0.3", default-features = false, features = ["serde", "std"] }
 either = { version = "1" }
 fail = { version = "0.5", default-features = false, features = ["failpoints"] }
 fixedbitset = { version = "0.4" }
+flate2 = { version = "1", features = ["zlib"] }
+frunk_core = { version = "0.4", default-features = false, features = ["std"] }
 futures = { version = "0.3" }
 futures-channel = { version = "0.3", features = ["sink"] }
 futures-core = { version = "0.3" }
@@ -54,12 +58,13 @@ lexical-write-integer = { version = "0.8", default-features = false, features =
 libc = { version = "0.2", features = ["extra_traits"] }
 lock_api = { version = "0.4", features = ["arc_lock"] }
 log = { version = "0.4", default-features = false, features = ["std"] }
-madsim-rdkafka = { git = "https://github.com/madsim-rs/madsim.git", rev = "f01e3ca", features = ["cmake-build", "gssapi", "ssl-vendored", "zstd"] }
+madsim-rdkafka = { git = "https://github.com/madsim-rs/madsim.git", rev = "bb8f063", features = ["cmake-build", "gssapi", "ssl-vendored", "zstd"] }
 madsim-tokio = { version = "0.2", default-features = false, features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 memchr = { version = "2" }
 miniz_oxide = { version = "0.7", default-features = false, features = ["with-alloc"] }
+mio = { version = "0.8", features = ["net", "os-ext"] }
 multimap = { version = "0.8" }
-nix = { version = "0.26" }
+nom = { version = "7" }
 num-bigint = { version = "0.4" }
 num-integer = { version = "0.1", features = ["i128"] }
 num-traits = { version = "0.2", features = ["i128", "libm"] }
@@ -86,9 +91,9 @@ scopeguard = { version = "1" }
 serde = { version = "1", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1", features = ["alloc"] }
 serde_with = { version = "3", features = ["json"] }
-smallvec = { version = "1", default-features = false, features = ["serde"] }
+smallvec = { version = "1", default-features = false, features = ["serde", "union", "write"] }
 subtle = { version = "2" }
-time = { version = "0.3", features = ["formatting", "local-offset", "macros", "parsing"] }
+time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] }
 tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] }
 tokio = { version = "1", features = ["full", "stats", "tracing"] }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "4538cd6", features = ["with-chrono-0_4"] }
@@ -103,6 +108,7 @@ unicode-bidi = { version = "0.3" }
 unicode-normalization = { version = "0.1" }
 url = { version = "2", features = ["serde"] }
 uuid = { version = "1", features = ["fast-rng", "serde", "v4"] }
+zeroize = { version = "1", features = ["zeroize_derive"] }
 
 [build-dependencies]
 ahash = { version = "0.8" }
@@ -111,6 +117,7 @@ auto_enums = { version = "0.8", features = ["futures03"] }
 aws-credential-types = { version = "0.55", default-features = false, features = ["hardcoded-credentials"] }
 aws-sdk-s3 = { version = "0.28", features = ["native-tls"] }
 aws-smithy-client = { version = "0.55", default-features = false, features = ["native-tls", "rustls"] }
+byteorder = { version = "1", features = ["i128"] }
 bytes = { version = "1", features = ["serde"] }
 cc = { version = "1", default-features = false, features = ["parallel"] }
 chrono = { version = "0.4", features = ["alloc", "serde"] }
@@ -119,9 +126,12 @@ clap_builder = { version = "4", default-features = false, features = ["cargo", "
 combine = { version = "4" }
 crossbeam-epoch = { version = "0.9" }
 crossbeam-utils = { version = "0.8" }
+deranged = { version = "0.3", default-features = false, features = ["serde", "std"] }
 either = { version = "1" }
 fail = { version = "0.5", default-features = false, features = ["failpoints"] }
 fixedbitset = { version = "0.4" }
+flate2 = { version = "1", features = ["zlib"] }
+frunk_core = { version = "0.4", default-features = false, features = ["std"] }
 futures = { version = "0.3" }
 futures-channel = { version = "0.3", features = ["sink"] }
 futures-core = { version = "0.3" }
@@ -143,12 +153,13 @@ lexical-write-integer = { version = "0.8", default-features = false, features =
 libc = { version = "0.2", features = ["extra_traits"] }
 lock_api = { version = "0.4", features = ["arc_lock"] }
 log = { version = "0.4", default-features = false, features = ["std"] }
-madsim-rdkafka = { git = "https://github.com/madsim-rs/madsim.git", rev = "f01e3ca", features = ["cmake-build", "gssapi", "ssl-vendored", "zstd"] }
+madsim-rdkafka = { git = "https://github.com/madsim-rs/madsim.git", rev = "bb8f063", features = ["cmake-build", "gssapi", "ssl-vendored", "zstd"] }
 madsim-tokio = { version = "0.2", default-features = false, features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time", "tracing"] }
 memchr = { version = "2" }
 miniz_oxide = { version = "0.7", default-features = false, features = ["with-alloc"] }
+mio = { version = "0.8", features = ["net", "os-ext"] }
 multimap = { version = "0.8" }
-nix = { version = "0.26" }
+nom = { version = "7" }
 num-bigint = { version = "0.4" }
 num-integer = { version = "0.1", features = ["i128"] }
 num-traits = { version = "0.2", features = ["i128", "libm"] }
@@ -176,11 +187,12 @@ scopeguard = { version = "1" }
 serde = { version = "1", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1", features = ["alloc"] }
 serde_with = { version = "3", features = ["json"] }
-smallvec = { version = "1", default-features = false, features = ["serde"] }
+smallvec = { version = "1", default-features = false, features = ["serde", "union", "write"] }
 subtle = { version = "2" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] }
 syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
-time = { version = "0.3", features = ["formatting", "local-offset", "macros", "parsing"] }
+time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] }
+time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] }
 tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] }
 tokio = { version = "1", features = ["full", "stats", "tracing"] }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "4538cd6", features = ["with-chrono-0_4"] }
@@ -195,5 +207,6 @@ unicode-bidi = { version = "0.3" }
 unicode-normalization = { version = "0.1" }
 url = { version = "2", features = ["serde"] }
 uuid = { version = "1", features = ["fast-rng", "serde", "v4"] }
+zeroize = { version = "1", features = ["zeroize_derive"] }
 
 ### END HAKARI SECTION